def __init__(self): Frame.__init__(self) self.master.title("ASL Game") self.master.minsize(width=500,height=500) self.grid() self.__dataPane = Frame(self) self.__dataPane.grid(row=0, column=0) self.__mode1Frame = Frame(self) self.__mode2Frame = Frame(self) ## self.__mode3Frame = Frame(self) self.__dp = DataProcessing() self.__whole = self.__dp.getWholeSignDict() self.__NumLtr = self.__dp.getNumLtrDict() self.__homeButton = Button(self.__dataPane, text="Home",\ command=self.__home) self.__welcome = Label(self.__dataPane, text="Choose a mode") self.__mode1Button = Button(self.__dataPane, text="Mode 1",\ command=self.__mode1) self.__mode2Button = Button(self.__dataPane, text="Mode 2",\ command=self.__mode2) ## self.__mode3Button = Button(self.__dataPane, text="Mode 3",\ ## command=self.__mode3) self.__homeButton.grid(row=1, column=1) self.__welcome.grid(row=2, column=2) self.__mode1Button.grid(row=3, column=1) self.__mode2Button.grid(row=3, column=2) #self.__mode3Button.grid(row=3, column=3) mainloop()
def plotByTTM(df, t, dim, subject_list, ylabel): k = dim[0]*dim[1] split_moneyness = DataProcessing.weightedSplit(df, k, 'moneyness') split_ttm = [] for moneyness in split_moneyness: split_ttm.append(DataProcessing.weightedSplit(moneyness, t, 'ttm')) fig = plt.figure() for subject in subject_list: plot_list = [] for moneyness in split_ttm: temp = [] for m in moneyness: temp.append([m['ttm'].median(), m[subject].mean()]) plot_list.append(np.array(temp)) count = 1 subplots = [] height = dim[1] for i in range(len(plot_list)): subplots.append(fig.add_subplot(int(str(height)+str(dim[0])+str(count)))) count += 1 for j in range(len(subplots)): subplots[j].plot(plot_list[j][:, 0], plot_list[j][:, 1]) subplots[j].set_title('Moneyness: '+str(round(split_moneyness[j]['moneyness'].min(), 3))+' - '+str(round(split_moneyness[j]['moneyness'].max(), 3))) subplots[j].set_xlabel('time-to-maturity') subplots[j].set_ylabel(ylabel) plt.show()
def Svmker(): x_train, x_test, y_train, y_test = train_test_split(dp.dataSet(2), dp.label(2), test_size=0.1, random_state=1) grid = [{'kernel': ['rbf'], 'gamma': [1e-2, 1e-4], 'C': [1, 10, 100]}] grid = GridSearchCV(SVC(), grid, cv=5) grid.fit(x_train, y_train) print("The best parameters are %s with a score of %0.2f" % (grid.best_params_, grid.best_score_))
def plotByMoneyness(df, k, dim, subject_list, ylabel): t = dim[0]*dim[1] split_ttm = DataProcessing.weightedSplit(df, t, 'ttm') split_moneyness = [] for ttm in split_ttm: split_moneyness.append(DataProcessing.weightedSplit(ttm, k, 'moneyness')) fig = plt.figure() subplots = [] plot_list = [] axins_list = [] for subject in subject_list: for ttm in split_moneyness: temp = [] for m in ttm: temp.append([m['moneyness'].median(), m[subject].mean()]) plot_list.append(np.array(temp)) height = dim[1] for i in range(t): subplots.append(fig.add_subplot(int(str(height)+str(dim[0])+str(i+1)))) # axins_list.append(zoomed_inset_axes(subplots[i], 2, loc=1)) subplots[i].set_title('Maturity: '+str(int(split_ttm[i]['ttm'].min()*365))+' - '+str(int(split_ttm[i]['ttm'].max()*365))+' days') subplots[i].set_xlabel('moneyness') subplots[i].set_ylabel(ylabel) count = 0 for j in range(len(subject_list)): # xmin = 1.025 for l in range(len(subplots)): subplots[l].plot(plot_list[count][:, 0], plot_list[count][:, 1], label=str(subject_list[j])) # axins_list[l].plot(plot_list[count][:, 0], plot_list[count][:, 1]) # axins_list[l].set_xlim(xmin, plot_list[count][:, 0].max()+0.01) # apply the x-limits # axins_list[l].set_ylim(plot_list[count][:, 1].min()-0.01, plot_list[count][:, 1][-1]+0.02) # apply the y-limits # axins_list[l].xaxis.set_visible(False) # axins_list[l].yaxis.set_visible(False) count += 1 # for p in range(len(axins_list)): # mark_inset(subplots[p], axins_list[p], loc1=3, loc2=1, fc="none", ec="0.5") # # axins_list[2].set_xlim(1., 1.1) # apply the x-limits # axins_list[3].set_xlim(1., 1.1) # apply the x-limits # axins_list[4].set_xlim(1., 1.1) # apply the x-limits # axins_list[5].set_xlim(1., 1.1) # apply the x-limits # axins_list[6].set_xlim(1., 1.1) # apply the x-limits # axins_list[7].set_xlim(1., 1.1) # apply the x-limits # axins_list[8].set_xlim(1., 1.2) # apply the x-limits plt.show()
def expected_time_to_trigger(bd, outdir): """ Represents the "hardness" of each triggered bug in a heatmap by computing the expected time to trigger for each bug :param bd: { A BenchmarkData object loaded from experiment summary file } :type bd: { BenchmarkData } """ ett, agg = DataProcessing.expected_time_to_trigger_data(bd) #Compute the order of the fuzzer that found the most bugs in descending order fuzzer_order = DataProcessing.number_of_unique_bugs_found_data(bd) fuzzer_order = fuzzer_order.sort_values(by=['Bugs'],ascending = False) \ .reset_index()['Fuzzer'].tolist() #Sort the bug by aggregate time ett['Aggregate'] = agg ett.sort_values(by='Aggregate', inplace=True) ett = ett.drop(labels='Aggregate', axis=1) #Reordering the fuzzers ett = ett[fuzzer_order] fuzzer_label = list(ett.columns) bug_label = list(ett.index) annotations = ett.copy() annotations[fuzzer_label] = annotations[fuzzer_label].applymap( lambda x: pp_time(x)) fig, ax = plt.subplots(figsize=(10, 10)) plt.yticks(rotation=0) #Norm factor has to been precomputed heat_map = sns.heatmap(np.array(ett), cmap='seismic', xticklabels=fuzzer_label, yticklabels=bug_label, annot=np.array(annotations), fmt='s', norm=colors.PowerNorm(gamma=0.32), ax=ax) #Color bar properties max_num_trials = bd.frame.reset_index().groupby('Fuzzer')['Campaign'] \ .nunique().max() xticks = list_ticks(bd.duration * max_num_trials)[4:] xticklables = list(map(lambda x: pp_time(x), xticks)) cbar = ax.collections[0].colorbar cbar.set_ticks(xticks) cbar.set_ticklabels(xticklables) ax.patch.set(fill='True', color='darkgrey') ax.xaxis.tick_top() name, path = output(outdir, 'data', 'summary_expected_ttb_heat.svg') fig.savefig(path) plt.close() return path
def main(): # 下载数据. raw_data = DataProcessing.download_data() # 数据编码及处理. data = DataProcessing.onehot_data(raw_data) data = data.values.astype(np.float32) # 将数据顺序打乱并将数据以8:2分为训练数据与测试数据. np.random.shuffle(data) scale = int(0.8 * len(data)) train_data = data[:scale] test_data = data[scale:] INPUT = tf.placeholder(tf.float32, [None, 25], "input") backward(train_data)
def confirm(self): dpr.add_item(self.data) if self.data[0]["Type"] == "Channel": dpr.update_videos(self.data[0]["ID"]) elif self.data[0]["Type"] == "Video": dpr.channel_from_video(self.data[0]["ID"]) dpr.refresh() self.close()
def test(): img = cv2.imread("./img/153g.bmp", 0) ip = ImageProcessing.ImageProcessing(img) ip.doHoughTrans() img = ip.cannyImg[ip.C - 125:ip.C + 125, :] waveHeight, waveLenght = dp.getWaveParaByPeak(img)
def train(): """ 训练模型 :return: """ du = DataProcessing.DataUnit(**data_config) save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME) steps = int(len(du) / batch_size) + 1 # 创建session的时候设置显存根据需要动态申请 tf.reset_default_graph() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.9 config.gpu_options.allow_growth = True with tf.Graph().as_default(): with tf.Session(config=config) as sess: # 定义模型 model = Seq2Seq(batch_size=batch_size, encoder_vocab_size=du.vocab_size, decoder_vocab_size=du.vocab_size, mode='train', **model_config) init = tf.global_variables_initializer() writer = tf.summary.FileWriter('./graph/nlp', sess.graph) sess.run(init) if continue_train: model.load(sess, save_path) model.export(sess) '''
def chatbot_api(infos): du = DataProcessing.DataUnit(**data_config) save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME) batch_size = 1 tf.reset_default_graph() model = Seq2Seq(batch_size=batch_size, encoder_vocab_size=du.vocab_size, decoder_vocab_size=du.vocab_size, mode='decode', **model_config) # 创建session的时候允许显存增长 config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: init = tf.global_variables_initializer() sess.run(init) model.load(sess, save_path) while True: q = infos if q is None or q.strip() == '': return "请输入聊天信息" continue q = q.strip() indexs = du.transform_sentence(q) x = np.asarray(indexs).reshape((1, -1)) xl = np.asarray(len(indexs)).reshape((1, )) pred = model.predict(sess, np.array(x), np.array(xl)) result = du.transform_indexs(pred[0]) return result
def runSpikeDetection(self): """runs stupid spike detector on only 1st channel""" channelIndex = self.canvas.indices[0] self.rawCh1Spikes = DataProcessing.stupidIdentifySpikes(self.canvas.rawData[channelIndex,:][0],cutoff=float(self.thresholdEdit.text())) self.ch1Spikes = self.canvas.rawData.index_as_time(self.rawCh1Spikes) self.canvas.annotationTimes = self.ch1Spikes[0] QtGui.QMessageBox.information(None,"Report","{d} spikes found.\n\n{i}".format(d=len(self.ch1Spikes[0]),i=self.ch1Spikes[0]))
def simulation(self, simulate, selected_nodes_R): """ The function to perform a simulation rollout (Delete a sequence of symbols). :param simulate: (ELLS-node) The start node for simulation. :param selected_nodes_R: (float) The total R-score that 'simulate' node already get. :return: (float) The total R-score of simulation. """ state = simulate.state actions = len(state) + self.delta - len(self.Z) simulation_reward = selected_nodes_R + self.ELLS_Total for i in range(actions): while 1: policy = random.randint(0, len(state) - 1) # The legal deleted characters cannot be the same as the neighbours. Such # method can ensure that the symbols deleted for this simulation node are # completely different each time. # Experimental result proves that random function is more effective than # searching for all legal deletions for each deletions. neighbor_right = policy + 1 if neighbor_right < len(state): if state[policy] == state[neighbor_right]: continue # Consider only the deletion that does not incur any sensitive patterns. reinstate = False after = state[_non_zero(policy - self.k + 1):policy] + state[policy + 1: policy + self.k] for j in range(len(after) - self.k + 1): if self._is_sensitve(after[j:j + self.k]): reinstate = True break if not reinstate: break simulation_reward += self._extract_r_score(state, policy) state = DataProcessing.delete(state, policy) return simulation_reward
def index(self): indoor_temp, outdoor_temp = DataProcessing.generate_graph() with open("/home/pi/Developpement/pyled/index.html") as file: html = '' for l in file: html += l return html.format(indoor_temp, outdoor_temp)
def onAmplitudeChanged(self, nAmplitude): self.storedAmplitude = nAmplitude; if(self.rawData): self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass,self.indices) self.signalData = np.float32(10000*np.array(self.displayData[0])) self.program['a_position'] = self.signalData.reshape(-1, 1) self.update()
def refresh(self): indoorTemp, outdoorTemp = DataProcessing.generate_complete_data() with open("/home/pi/Developpement/pyled/current_temp.txt", 'w') as file: file.write(str(indoorTemp) + " " + str(outdoorTemp)) print(indoorTemp, outdoorTemp) raise cherrypy.HTTPRedirect("/")
def data_preprocess(self): # 正运动学求取末端位姿 X = np.zeros([self.num, 6]) for i in range(self.num): X[i, :] = kin.fkine_euler(self.DH_0, self.qq_array[i, :]) #求取末端位置的,前向差分获取速度和加速度 X_xva = np.zeros([self.num, self.m, 3]) X_xva[:, :, 0] = X #建立高增益观测器 observer = dp.HighGainObserver() for i in range(self.m): observer.get_original_data(X[:, i], self.T) X_xva[:, i, 1] = observer.put_observer_data() observer.get_original_data(X_xva[:, i, 1], self.T) X_xva[:, i, 2] = observer.put_observer_data() # for i in range(self.num): # X_xva[i, :, 0] = X[i, :] # if (i==0): # X_xva[i, :, 1] = 0 # X_xva[i, :, 2] = 0 # else: # X_xva[i, :, 1] = (X_xva[i, :, 0] - X_xva[i-1, :, 0])/self.T # X_xva[i, :, 2] = (X_xva[i, :, 1] - X_xva[i-1, :, 1])/self.T self.X_xva = X_xva
def _init_funcs(self, number=None, blacklist_defaults=True, blacklist_forms=True, csvdata=None): '''initial functions, once self.files is set.''' if number is not None: self.number = number else: if csvdata is not None: import DataProcessing as dp self.number = dp.poke_to_N(self.pokemon, csvdata) else: self.number = None if self.number is not None: self.number = str(self.number).zfill(3) took_pre = [take_pre(s) for s in self.files] self.pre = [x[0] for x in took_pre] self.sites = [x[1] for x in took_pre] took_ext = [take_ext(s) for s in self.sites] self.ext = [x[1] for x in took_ext] self.names = [x[0] for x in took_ext] self.comps = [x.split('_') for x in self.names] self._idx = np.arange(len(self.files)) self.idx = np.arange(len(self.files)) if blacklist_defaults: self.blacklist_defaults() if blacklist_forms: self.blacklist_forms()
def __init__(self, keyname, training_data): """ Initialize the TrainingSet class. This class contains Session objects, which represent single sessions of training data on a user-by-user basis. Session objects are passed with their member DataPoints to the DataProcessing module where they can be analyzed and manipulated. :keyname represents the name of the indepdendent variable for which we will try to make predictions. :training_data represents the file contents of the saved session files, passed to the initializer as a list of file contents. """ # Create a new featureset object and pass it the name of the prediction target variable self.featureset = Featureset(keyname) self.classify_on_key = keyname self.training_data = training_data # Kick off the training session by actually creating session objects in self.sessions if self.parse() is True: # Now create Data Processing objects and pass all of the sessions and features self.processor = DataProcessing.DataProcessor( self.featureset, self.sessions) # Do more things with the DataProcessor... else: sys.stderr.write("Error parsing data: exiting...")
def draw_bbox_video(image, data, g_cerca, g_lejos): color_red = (0, 0, 255) color_green = (0, 255, 0) ind_p, p_bbox_center, pair_point_bbox = datap.process_df2bbox(data) for pair_points in g_cerca: points_line = [] for points in pair_points: id_people = points[2] ind_list = ind_p.index(id_people) cv2.rectangle(img=image, pt1=point_float2int(pair_point_bbox[ind_list][0]), pt2=point_float2int(pair_point_bbox[ind_list][1]), color=color_red, thickness=2) points_line.append(p_bbox_center[ind_list]) cv2.line(image, points_line[0], points_line[1], color_red, thickness=2) for Po in g_lejos: id_people = Po[2] ind_list = ind_p.index(id_people) cv2.rectangle(img=image, pt1=point_float2int(pair_point_bbox[ind_list][0]), pt2=point_float2int(pair_point_bbox[ind_list][1]), color=color_green, thickness=2) return image
def baseline(self): """ The function of the baseline algorithm. Firstly, sort the distortion of each pattern in descending order. Second, traverse each letter from the most distorted pattern to the least distorted pattern. The loop will terminate if and only if the current candidate letter does not incur any sensitive pattern. :return BA: (String) the string type of data, whose length is |Z| - delta. """ Z = self.Z w_nsens = self.non_sens_w z_nsens = self._search_nsens(Z) for i in range(self.delta): # A list of all candidate patterns' distortions in Z candidates = [(abs(z_nsens[Z[j:j + self.k]] - w_nsens[Z[j:j + self.k]]), list(range(j, j + self.k))) for j in range(len(Z) - self.k + 1)] # Sort distortions in descending order. # The most distorted pattern will be considered first. candidates.sort(key=lambda x: x[0], reverse=True) for candidate in candidates: for victim in candidate[1]: after = Z[_non_zero(victim - self.k + 1):victim] + Z[victim + 1: victim + self.k] non_sens_after = self._search_nsens(after) # check if 'victim' led to any sensitive patterns. if len(set(non_sens_after.keys()).intersection(set(self.S))) == 0: Z = DataProcessing.delete(Z, victim) z_nsens = self._search_nsens(Z) break else: continue break else: continue break return Z
def TrimData(data: pd.DataFrame, linearization_point: LinearizationPoint, data_constraints: dict = None): """ dataConstraints is a map of data key to maximum allowable deviation """ if data_constraints is not None: if type(linearization_point) is LinearizationPoint: lp_dict = linearization_point._asdict() else: lp_dict = linearization_point.__dict__ data = DataProcessing.constrain_data(data, lp_dict, data_constraints) v = data['v'] - linearization_point.v p = data['p'] r = data['r'] phi = data['phi'] - linearization_point.phi vdot = data['v_dot'] pdot = data['p_dot'] rdot = data['r_dot'] aileron = data['roll_ctrl'] - linearization_point.aileron_ctrl rudder = data['yaw_ctrl'] - linearization_point.rudder_ctrl return TrimmedData(v, p, r, phi, vdot, pdot, rdot, aileron, rudder)
def learn_boosted(tdepth, nummodels, datapath): train_location = datapath + '/agaricuslepiotatrain1.csv' f = open(train_location, 'r') data = DataProcessing.preprocess(f) test_location = datapath + '/agaricuslepiotatest1.csv' t = open(test_location, 'r') tdata = DataProcessing.preprocess(t) boosting_models = boosting(data, nummodels, tdepth) accuracy = classify_accu(boosting_models, tdata) print("Accuracy : ", accuracy)
def onTextBoxesChanged(self, lowPass, highPass): self.lowPass = lowPass; self.highPass = highPass; print "startTime: {s}, endTime: {e}, lowPass: {l}, highPass: {h}".format(s=self.startTime, e=self.endTime, l=self.lowPass, h=self.highPass) self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass,self.indices) self.signalData = np.float32(10000*np.array(self.displayData[0])) self.program['a_position'] = self.signalData.reshape(-1, 1) self.update()
def SaveDataToFile(): try: objF = DataProcessing.DataProcessing() objF.FileName = "CustomerData.txt" objF.TextData = Customers.CustomerList.ToString() objF.SaveData() except Exception as e: print(e)
def __init__(self, type, input_file): filename = dpr.get_datafilepath(input_file) #create a list with IDs of all channels or all videos, so we can loop through them later with open(filename, "r") as f: self.reader = csv.reader(f, delimiter="\t") self.id_list = [rows[0] for rows in self.reader] self.id_list.pop(0) #remove header row self.type = type #save instance as either channel or video object
def Operate(Specs): dataObj = DataProcessing.DataProcessing(Specs) # TBD make sure that input data is valid etc. # TBD back test results against real data # TBD Error handling # TBD fix model paths #print("----------------------------------------------------------") #print("event forecast started.") #print(datetime.now()) #print("----------------------------------------------------------") if not hasattr(dataObj, "data"): print("no valid data could be extracted.") return if Specs.Mode == "2": print("No Neural Network support at the moment.") return if not int(Specs.AnzahlPrognosen) == 1 or not Specs.AnzahlPrognosen: print( "corrected wrong configuration: Only realtime forecast is considered" ) Specs.AnzahlPrognosen = 1 print("input data: ", dataObj.datapath) #load model try: mdlObj = generateModel(Specs, dataObj) except Exception as e: print("problems while loading model: " + e) try: ForecastDyn = mdlObj.predictOperative(Specs.horizont) timeseriesNF = dataObj.data["NoFilter"] except Exception as e: print("problems while prediction:\n" + e) try: print("creating csv.") createCsv(ForecastDyn, Specs) print("done") except Exception as e: print("no csv could be created:\n" + e) try: DataDyn = timeseriesNF[-Specs.horizont:] multi = "\n##### " + str( Specs.horizont) + "-step Prediction against last values ##### \n" print(multi) sys.stdout.flush() FDyn = ErrorAnalysis.ErrorAnalysis(DataDyn, ForecastDyn) print(FDyn.criterias) sys.stdout.flush() except Exception as e: print(e)
def _ELLS_ALGO(self, root, iterations): """ The main function of ELLS-ALGO. This function recursively finding the most promising symbol to delete by running a certain number of Monte Carlo cycles (Selection -> Expansion -> Simulation -> Backpropagation) until all optimal delete characters. :param root: (ELLS_node) The root node of the tree stores the string to be deleted. :param iterations: (int) The number of deletions required to make a decision(select deleted symbol) :return: H:(string) The resulting data after deletions. """ # Base case of recursion: recursion terminates as only one remaining symbol need to be deleted. if self.remain_delete == 1: score = self._get_legal_deletions(root.state, strategy='all') # greedy find current best deleted symbol self.ELLS_Total += score[0] result = DataProcessing.delete(root.state, score[1]) self.ELLS_track.append(result) return result # Expand the root node beforehand. if not root.isExpanded: root.expand(self._get_legal_deletions(root.state), self.C) for _ in range(iterations): # Selection simulate, selected_nodes_R = root.select_leaf() # Expansion delay if simulate.visits > 3 and len(self.Z) - len(simulate.state) < self.delta: # Expansion simulate.expand(self._get_legal_deletions(simulate.state), self.C) simulate = simulate.children[next(iter(simulate.children))] simulate_score = selected_nodes_R + simulate.score else: simulate_score = selected_nodes_R # Simulation simulation_reward = self.simulation(simulate, simulate_score) # Backpropagation simulate.backpropagation(simulation_reward) # Final deleted symbol selection -> select the node by max-robust child selection policy. best_child = None best_result = - float('inf') for child in root.children.values(): score = - child.reward + child.visits if score > best_result: best_result = score best_child = child self.ELLS_track.append(best_child.state) # Search tree reuse iterations = self.max_simulations - np.sum([child.visits for child in best_child.children.values()]) self.remain_delete -= 1 self.ELLS_Total += best_child.score best_child.refresh() gc.collect() # transform best root child to next tree's root node. return self._ELLS_ALGO(best_child, int(iterations))
def buttonClicked(self): sid = int(self.ui.comboBox.currentText().split("-")[0].strip()) year = int(self.ui.comboBox_2.currentText()) season = int(self.ui.comboBox_3.currentText()) self.df = DataProcessing.crwal_financial_report(sid, year, season) model = PandasModel(self.df) self.ui.tableView.setModel(model)
def update_stats(self, output_file, stat): #create df_stats with only relevant data(1 column - date = name) self.curr_time = time.strftime("%Y-%m-%dT%H:%M:%S.000Z") #try to match the IDs of all videos or channels with the IDs of the data from gather_data. self.stats = {} for i in self.id_list: try: self.stats[i] = self.data[i][stat] except KeyError as e: self.stats[i] = "NaN" self.df_stats = pd.DataFrame.from_dict(self.stats, orient="index", columns=[self.curr_time]) #if there is no previous channels- or videos-file or if they are empty, don't merge with df_stats before writing try: self.df_old = dpr.get_dataframe(output_file) self.df_new = self.df_old.join(self.df_stats, how="outer") dpr.write_dataframetofile(self.df_new, output_file) except (ValueError, OSError): dpr.write_dataframetofile(self.df_stats, output_file)
def SaveDatatoFile(): try: ObjFile = DataProcessing.File() ObjFile.TextFile = "CustomerDetails.txt" ObjFile.TextData = Customers.CustomerList.ToString() print("it worked, Done!") ObjFile.SaveData() except Exception as e: print(e)
def trainTwoboxModel(modelName, model, epochs, modelPath="SavedModels"): trainX, trainY = dp.dataPreprocessing(training_twobox) testX, testY = dp.dataPreprocessing(testing_twobox) modelPath += "twobox_" + modelName if not os.path.exists(modelPath): os.mkdir(modelPath) model.fit(trainX, trainY, n_epoch=epochs, shuffle=True, validation_set=(testX, testY), show_metric=True, batch_size=BATCH_SIZE, run_id=modelName + "_twobox") model.save(modelPath + '/' + modelName + '.tfl')
def SaveDataToFile(): try: objF = DataProcessing.DataProcessing() objF.FileName = "EmployeeData.txt" objF.TextData = Employees.EmployeeList.ToString() print("Reached here") objF.SaveData() except Exception as e: print(e)
def onStartEndChanged(self, startTime, endTime): self.startTime = startTime; self.endTime = endTime; self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass,self.indices) self.setupZoom(self.displayData) # self.program.delete() # self.program = gloo.Program(SERIES_VERT_SHADER, SERIES_FRAG_SHADER) self.setupZoomStep2() self.updateLines() self.update()
def onAmplitudeChanged(self, nAmplitude): self.storedAmplitude = nAmplitude if (self.rawData): self.displayData = DataProcessing.getDisplayData( self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass, self.indices) self.signalData = np.float32(10000 * np.array(self.displayData[0])) self.program['a_position'] = self.signalData.reshape(-1, 1) self.update()
def test_2_getXY(self): XY = DataProcessing.getXY(MERGEDA, "HC_AD") self.assertEqual(np.array_equal(XY[1], [1,1,1,0,0,0]), True) self.assertEqual(np.array_equal(XY[1], [1,1,1,1,0,0]), False) XY = DataProcessing.getXY(MERGEDA, "MCI_AD") self.assertEqual(np.array_equal(XY[1], [1,1,1,0,0,0,0]), True) self.assertEqual(np.array_equal(XY[1], [1,1,1,1,0,0,0]), False) XY = DataProcessing.getXY(MERGEDA, "HC_MCI") self.assertEqual(np.array_equal(XY[1], [0,0,0,1,1,1,1]), True) self.assertEqual(np.array_equal(XY[1], [0,0,0,0,1,1,1]), False) self.assertEqual(np.array_equal(XY[2], ["RightAmygdala", "RightHippocampus"]), True) self.assertEqual(np.array_equal(XY[2], ["RightHippocampus", "RightAmygdala"]), False) self.assertEqual(np.array_equal(XY[4], MERGEDA), False) print("getXY with MERGED A tested")
def __init__(self, df): super(TableFromDf, self).__init__() self.df = df self.df_prep = dpr.prep_table_dsp(df) self.setColumnCount(len(self.df_prep.columns)) self.setRowCount(len(self.df_prep.index)) self.setShowGrid(False) self.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows) self.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers) self.populate(self.df_prep)
def train(): """ 训练模型 :return: """ du = DataProcessing.DataUnit(**data_config) save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME) trainset, _, _, _ = pickle.load(open('dataset.pkl', 'rb')) steps = int(len(list(trainset)) / batch_size) + 1 print(trainset.shape) # 创建session的时候设置显存根据需要动态申请 tf.reset_default_graph() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.9 config.gpu_options.allow_growth = True with tf.Graph().as_default(): with tf.Session(config=config) as sess: # 定义模型 model = Seq2Seq(batch_size=batch_size, encoder_vocab_size=du.vocab_size, decoder_vocab_size=du.vocab_size, mode='train', **model_config) init = tf.global_variables_initializer() sess.run(init) if continue_train: model.load(sess, save_path) loss_list = [] for epoch in range(1, n_epoch + 1): costs = [] bar = tqdm(range(steps), total=steps, desc='epoch {}, loss=0.000000'.format(epoch)) for time in bar: x, xl, y, yl = du.next_batch(batch_size, list(trainset)) max_len = np.max(yl) y = y[:, 0:max_len] cost, lr = model.train(sess, x, xl, y, yl, keep_prob) costs.append(cost) if epoch == 1 and time == 0: loss_list.append(np.mean(costs)) if time == steps - 1: loss_list.append(np.mean(costs)) bar.set_description( 'epoch {} loss={:.6f} lr={:.6f}'.format( epoch, np.mean(costs), lr)) epoch_list = list(range(0, n_epoch + 1)) plt.xlabel('Epoch Number') plt.ylabel('Loss Value') plt.plot(epoch_list, loss_list) plt.show() model.save(sess, save_path=save_path)
def main(): # TODO preprocess the input file to get standard vectors configuration = config.get_config() filepath = configuration['datafile_path'] processed_data = DataProcessing.ProcessData(filepath) """ Model designing part """ # TODO design encoder for epi in range(configuration['max_epochs']): # print "training epoch ", epi # err = 0.0 num_steps = 0 # TODO: shuffle the training data and train this epoch ## train_start = time.time() # seq_lang_numpy = [] seq_world_numpy = [] seq_action_numpy = [] for name_map in configuration['maps_train']: max_steps = len( processed_data.dict_data['train'][name_map] ) print 'max_steps=', max_steps for idx_data, data in enumerate(processed_data.dict_data['train'][name_map]): # seq_lang_numpy, seq_world_numpy and seq_action_numpy will be set seq_lang_numpy, seq_world_numpy, seq_action_numpy = processed_data.process_one_data(idx_data, name_map, 'train') # np.concatenate((seq_lang_numpy, seq_lang)) # np.concatenate((seq_world_numpy, seq_world)) # np.concatenate((seq_action_numpy, seq_action)) """ trainer = Instantiates the model """ model = models.SeqToSeq() cost_numpy = model.build_model( seq_lang_numpy, # list of word indices seq_world_numpy, # matrix of dim (len(one_data['cleanpath'])*78 seq_action_numpy # index value of 1 in one hot vector of action ) print "Cost!!------", cost_numpy print "type = ", type(cost_numpy) print "shape = ", cost_numpy.shape print "---Cost_numpy___=",cost_numpy err += cost_numpy if idx_data % 100 == 99: print "training i-th out of N in map : ", (idx_data, max_steps, name_map) # num_steps += max_steps # train_err = err / num_steps
def main(): import DataProcessing as dp # This is my customized module for data pre-process, can be found here : https://github.com/newtoncircus/DEXTRA_SourceCodes/blob/master/DataProcessing.py root_path = "./" # path_predicted = "SampleSubmission_percent10.csv" # path_actual = "public_validation_datasheet_percent10.csv" path_predicted = "predicted.csv" path_actual = "true.csv" viki = dp.rakuten() (header_sample, arr_predicted) = viki.csv_to_array(root_path + path_predicted, "rU") (header_public, arr_actual) = viki.csv_to_array(root_path + path_actual, "rU") num = 3 EWAP(arr_predicted, arr_actual, num)
def btn_add_video_press(self): self.video_data = dpr.check_video(self.line_add_url.text()) if type(self.video_data) is str: self.video_add_popup = errorDialog("Error") self.video_add_popup.display_text.setText(self.video_data) self.video_add_popup.exec_() elif type(self.video_data) is list: self.video_add_popup = confirmDialog("Add Video", self.line_add_url.text(), self.video_data) self.video_add_popup.display_text.setText( "Is \"" + self.video_data[0]["Title"] + "\" your video?") self.video_add_popup.exec_() else: print("wrong return format")
def handle_scroll(self, fdx): dx = -fdx#-np.sign(event.delta[1]) if((self.startTime + dx) > 0.5): self.startTime += dx self.endTime += dx elif ((self.startTime+dx) >= 0.0): #CB handle edge cases as we get close to beginning of dataset self.startTime += dx self.endTime += dx olen=len(self.displayData[0][0]) self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass,self.indices) if(len(self.displayData[0][0]) != olen): self.onStartEndChanged(self.startTime,self.endTime) self.signalData = np.float32(10000*np.array(self.displayData[0])) self.program['a_position'] = self.signalData.reshape(-1, 1) self.updateTextBoxes() self.updateLines() self.update()
def __init__(self, channel): QWidget.__init__(self) self.id = channel.name self.setWindowIcon(QtGui.QIcon('app.png')) self.setWindowTitle("Channel Details") self.vBox_1 = QtWidgets.QVBoxLayout(self) self.hBox_top = QtWidgets.QHBoxLayout() self.lbl_header = QtWidgets.QLabel(channel["Title"], font=f_header) self.hBox_top.addWidget(self.lbl_header) self.hBox_top.addStretch() self.btn_delete = QtWidgets.QPushButton("Delete Channel", self) self.btn_delete.clicked.connect(self.delete_channel) self.hBox_top.addWidget(self.btn_delete) self.vBox_1.addLayout(self.hBox_top) self.hBox_graphs = QtWidgets.QHBoxLayout() if dpr.datafile_exists("channel_subscribers.csv") and dpr.datafile_exists("channels.csv"): self.gra_channel_subs = LineGraphCanvas( dpr.get_subs_ot(self.id),"Subscribers over Time") self.hBox_graphs.addWidget(self.gra_channel_subs) self.gra_channel_subs_growth = LineGraphCanvas( dpr.get_growth(dpr.get_subs_ot(self.id)),"Subscriber Growth over Time") self.hBox_graphs.addWidget(self.gra_channel_subs_growth) self.vBox_1.addLayout(self.hBox_graphs) self.hBox_details = QtWidgets.QHBoxLayout() self.channel_details = channelDetails(channel) self.hBox_details.addLayout(self.channel_details) self.vBox_video_details = QtWidgets.QVBoxLayout() self.lbl_video_header = QtWidgets.QLabel("Last Uploads", font=f_header) self.vBox_video_details.addWidget(self.lbl_video_header) self.videos = dpr.get_scanned_videos(self.id, df_videos) self.recent_vids = dpr.get_sorted(self.videos, "Upload", 3) self.video_details = [videoDetails(self.recent_vids.iloc[x]) for x in range(len(self.recent_vids))] for i in range(len(self.video_details)): self.vBox_video_details.addLayout(self.video_details[i]) self.hBox_details.addLayout(self.vBox_video_details) self.hBox_details.addStretch() self.vBox_1.addLayout(self.hBox_details) self.vBox_1.addStretch()
def __init__(self, video): QWidget.__init__(self) print(video.name) self.id = video.name self.channel_id = dpr.get_channel_id(self.id) self.setWindowIcon(QtGui.QIcon('app.png')) self.setWindowTitle("Video Details") self.vBox_1 = QtWidgets.QVBoxLayout(self) self.hBox_top = QtWidgets.QHBoxLayout() self.lbl_header = QtWidgets.QLabel(video["Title"], font=f_header) self.hBox_top.addWidget(self.lbl_header) self.hBox_top.addStretch() self.btn_delete = QtWidgets.QPushButton("Delete Video", self) self.btn_delete.clicked.connect(self.delete_video) self.hBox_top.addWidget(self.btn_delete) self.vBox_1.addLayout(self.hBox_top) self.hBox_graphs = QtWidgets.QHBoxLayout() if dpr.datafile_exists("video_views.csv") and dpr.datafile_exists("videos.csv"): self.gra_video_views = LineGraphCanvas( dpr.get_views_ot(self.id),"Views over Time") self.hBox_graphs.addWidget(self.gra_video_views) self.gra_video_views_growth = LineGraphCanvas( dpr.get_growth(dpr.get_views_ot(self.id)),"Growth of Views over Time") self.hBox_graphs.addWidget(self.gra_video_views_growth) self.vBox_1.addLayout(self.hBox_graphs) self.hBox_details = QtWidgets.QHBoxLayout() self.video_details = videoDetails(video) self.hBox_details.addLayout(self.video_details) self.channel_details = channelDetails(df_channels.loc[self.channel_id]) self.hBox_details.addLayout(self.channel_details) self.hBox_details.addStretch() self.vBox_1.addLayout(self.hBox_details) self.vBox_1.addStretch()
else: print("ERROR: confirmDeleteDialog wrong type") self.btn_confirm = QtWidgets.QPushButton("Yes", self) self.btn_confirm.clicked.connect(self.confirm) self.btn_cancel = QtWidgets.QPushButton("No", self) self.btn_cancel.clicked.connect(self.exit) self.hBox_1 = QtWidgets.QHBoxLayout() self.hBox_1.addWidget(self.btn_confirm) self.hBox_1.addWidget(self.btn_cancel) self.vBox_1 = QtWidgets.QVBoxLayout(self) self.vBox_1.addWidget(self.display_text) self.vBox_1.addLayout(self.hBox_1) def confirm(self): if self.type == "Video": dpr.delete_video(self.id) elif self.type == "Channel": dpr.delete_channel(self.id) self.close() def exit(self): self.close() if __name__ == "__main__": df_videos = dpr.get_all("Videos") df_channels = dpr.get_all("Channels") app = QApplication(sys.argv) ex = MainFrame() ex.show() sys.exit(app.exec_())
#wx is a linear function of the weights and input wx = w * record.transpose() #hypothesis, h is true if wx is positive, else it is false if wx >= 0: h = 1 else: h = 0 #if hpyothesis and target labels don't match, then the example has been misclassified if h != y: misclassified.append(row) if len(misclassified) != 0: #pick random misclassified point point = choice(misclassified) record = [data.item(point, i) for i in range(len(attr))] w = w + np.matrix(record[target_attr]) * np.matrix(record[0:len(attr)]) count += 1 else: #no misclassifications, Perceptron has attained convergence converged = 1 if not converged: print("No convergence") if __name__ == "__main__": data = np.matrix(DataProcessing.process("data.txt","processedData.txt")) perceptron(data, range(14), 13)
def setupDataDisplay(self, indices=range(1,15)): """requires that you have already set a number of things on self""" self.indices = indices self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass, self.indices) self.setupZoom(self.displayData) self.channels = self.rawData.ch_names self.samplingRate = self.rawData.info['sfreq'] displayChannels = [self.channels[i] for i in indices] displayPositions = np.linspace(0.9,-0.9,len(displayChannels)) self.annotationTimes = [] self.positionsToTextMap = {} for i in range(0,len(displayPositions)): self.positionsToTextMap[(-0.97,displayPositions[i])]=str(displayChannels[i]) self.program = gloo.Program(SERIES_VERT_SHADER, SERIES_FRAG_SHADER) self.vertices = gloo.VertexBuffer(V) self.zoomBoxBuffer = gloo.VertexBuffer(V3) self.progAnnotations = gloo.Program(progAnnotations_vertex_shader, progAnnotations_fragment_shader) self.progAnnotations.bind(self.vertices) self.progZoom = gloo.Program(zoombox_vertex_shader, zoombox_fragment_shader) self.progZoom.bind(self.zoomBoxBuffer) self.setupZoomStep2() self.myTextDrawer = TextDrawer(self.physical_size[1],self.physical_size[0]) self.myLineDrawer = LineDrawer() print "height " + str(self.physical_size[1]) self.progText = gloo.Program(text_vertex_shader, text_fragment_shader) self.fontBMP= vispy.io.imread("FixedSys.bmp") tvar = np.array(self.fontBMP[:,:,0]) self.fontBMP=np.dstack((self.fontBMP,tvar)) self.fontTexture =gloo.Texture2D(self.fontBMP,format="rgba") self.textVerticesArr = self.myTextDrawer.computeTextsData(self.positionsToTextMap) self.updateLines() self.textVertices = gloo.VertexBuffer(self.textVerticesArr) self.progText.bind(self.textVertices) self.progText["myTextureSampler"] = self.fontTexture gloo.set_viewport(0, 0, *self.physical_size) self.updateTextBoxes() self.dragZoom = False self.oldPos = None self.newPos = None self.orig = None self.new = None self.posDiff = None self.events.mouse_press.connect((self, 'mouse_press')) self.events.mouse_release.connect((self, 'mouse_release')) self.events.mouse_move.connect((self, 'on_mouse_move')) # self._timer = app.Timer('auto', connect=self.on_timer, start=True) gloo.set_state(clear_color='black', blend=True, blend_func=('src_alpha', 'one_minus_src_alpha')) self.update()
def setupChannels(self): self.tab_channels = QtWidgets.QStackedWidget() self.tab_channels.setObjectName("tab_channels") #--- Channel Stats self.stk_channel_stats = QtWidgets.QWidget() self.vBox_channel_stats_1 = QtWidgets.QVBoxLayout(self.stk_channel_stats) self.hBox_channel_stats_1 = QtWidgets.QHBoxLayout() if dpr.datafile_exists("channel_views.csv") and dpr.datafile_exists("channels.csv"): self.gra_channel_subs = LineGraphCanvas( dpr.get_sums_ot(r"channel_subscribers.csv"), "Reached Subscribers Over Time") self.hBox_channel_stats_1.addWidget(self.gra_channel_subs) self.gra_channels_count = LineGraphCanvas( dpr.get_num_items_ot(r"channel_subscribers.csv"), "Number of Channels Over Time") self.hBox_channel_stats_1.addWidget(self.gra_channels_count) self.vBox_channel_stats_1.addLayout(self.hBox_channel_stats_1) self.btn_goto_channel_list = QtWidgets.QPushButton("Channel List", self.stk_channel_stats) self.btn_goto_channel_list.clicked.connect(self.goto_channel_list) self.vBox_channel_stats_1.addWidget(self.btn_goto_channel_list) self.tab_channels.addWidget(self.stk_channel_stats) #--- Channel List self.stk_channel_list = QtWidgets.QWidget() self.hBox_channel_list_1 = QtWidgets.QHBoxLayout(self.stk_channel_list) self.hBox_channel_list_1.setObjectName("hBox_channel_list_1") self.vBox_channel_list_1 = QtWidgets.QVBoxLayout() self.vBox_channel_list_1.setObjectName("vBox_channel_list_1") self.hBox_channel_list_2 = QtWidgets.QHBoxLayout() self.hBox_channel_list_2.setObjectName("hBox_channel_list_2") self.lbl_channel_list_popularity = QtWidgets.QLabel("Sort:") self.hBox_channel_list_2.addWidget(self.lbl_channel_list_popularity) self.cmb_channel_list_popularity = QtWidgets.QComboBox(self.stk_channel_list) self.cmb_channel_list_popularity.setObjectName("cmb_channel_list_popularity") self.channel_popularity = ["Subscribers", "Most Recent", "Channel Name"] for i in self.channel_popularity: self.cmb_channel_list_popularity.addItem(i) self.hBox_channel_list_2.addWidget(self.cmb_channel_list_popularity) self.cmb_channel_list_popularity.currentIndexChanged.connect(self.cmb_channel_popularity) spacerItem = QtWidgets.QSpacerItem( 40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum) self.hBox_channel_list_2.addItem(spacerItem) self.vBox_channel_list_1.addLayout(self.hBox_channel_list_2) self.tbl_channel_list = TableFromDf(df_channels) self.tbl_channel_list.setObjectName("tlb_channel_list") self.tbl_channel_list.itemDoubleClicked.connect(self.tbl_show_channel) self.vBox_channel_list_1.addWidget(self.tbl_channel_list) self.btn_goto_channel_stats = QtWidgets.QPushButton( "Channel Stats", self.stk_channel_list) self.btn_goto_channel_stats.clicked.connect(self.goto_channel_stats) self.vBox_channel_list_1.addWidget(self.btn_goto_channel_stats) self.hBox_channel_list_1.addLayout(self.vBox_channel_list_1) self.tab_channels.addWidget(self.stk_channel_list)
def cmb_video_region(self, index): dpr.filter_video_region(self.cmb_video_list_region.itemText(index))
def cmb_video_popularity(self, index): self.tbl_video_list.populate(dpr.prep_table_dsp(dpr.sort_videos(self.cmb_video_list_popularity.itemText(index), self.tbl_video_list.df))) self.tbl_video_list.df = dpr.sort_videos(self.cmb_video_list_popularity.itemText(index), self.tbl_video_list.df)
def cmb_channel_region(self, index): dpr.filter_channel_region(self.cmb_channel_list_region.itemText(index))
import DataProcessing as dp import network as NN train = dp.get_data_train("mnist_train.csv") test = dp.get_data_test("mnist_test.csv") print "*************************" print "* train and test loaded *" print "*************************" nn = NN.Network([784,30,10]) print "*************************" print "* nn loaded *" print "*************************" nn.SGD(train,30,10,0.1,test_data=test)
def __init__(self, video): QHBoxLayout.__init__(self) self.id = video.name self.vid_thumbdata = dpr.get_thumb_data(self.id, "Video") self.img_vid = QtGui.QPixmap() self.img_vid.loadFromData(self.vid_thumbdata) self.img_vid = self.img_vid.scaledToHeight(80) self.lbl_vid = QtWidgets.QLabel() self.lbl_vid.mouseReleaseEvent = self.goToVideo self.lbl_vid.setPixmap(self.img_vid) self.addWidget(self.lbl_vid) self.vBox_all = QtWidgets.QVBoxLayout() self.vBox_all.addStretch() self.lbl_name = QtWidgets.QLabel(video["Title"], font=f_subheader) self.vBox_all.addWidget(self.lbl_name) self.hBox_top = QtWidgets.QHBoxLayout() self.vBox_views = QtWidgets.QVBoxLayout() self.lbl_views = QtWidgets.QLabel("Views") self.vBox_views.addWidget(self.lbl_views) self.lbl_views_val = QtWidgets.QLabel(str(video["Views"])) self.vBox_views.addWidget(self.lbl_views_val) self.hBox_top.addLayout(self.vBox_views) self.vBox_channel = QtWidgets.QVBoxLayout() self.lbl_channel = QtWidgets.QLabel("Channel") self.vBox_channel.addWidget(self.lbl_channel) self.lbl_channel_val = QtWidgets.QLabel(video["ChannelTitle"]) self.vBox_channel.addWidget(self.lbl_channel_val) self.hBox_top.addLayout(self.vBox_channel) self.vBox_likes = QtWidgets.QVBoxLayout() self.lbl_likes = QtWidgets.QLabel("Likes/Dislikes") self.vBox_likes.addWidget(self.lbl_likes) self.lbl_likes_val = QtWidgets.QLabel(str(video["Likes"])+"/"+str(video["Dislikes"])) self.vBox_likes.addWidget(self.lbl_likes_val) self.hBox_top.addLayout(self.vBox_likes) self.vBox_duration = QtWidgets.QVBoxLayout() self.lbl_duration = QtWidgets.QLabel("Duration") self.vBox_duration.addWidget(self.lbl_duration) self.lbl_duration_val = QtWidgets.QLabel(str(dpr.parse_duration(video["Duration"]))) self.vBox_duration.addWidget(self.lbl_duration_val) self.hBox_top.addLayout(self.vBox_duration) self.vBox_comments = QtWidgets.QVBoxLayout() self.lbl_comments = QtWidgets.QLabel("Comments") self.vBox_comments.addWidget(self.lbl_comments) self.lbl_comments_val = QtWidgets.QLabel(str(video["Comments"])) self.vBox_comments.addWidget(self.lbl_comments_val) self.hBox_top.addLayout(self.vBox_comments) self.vBox_upload = QtWidgets.QVBoxLayout() self.lbl_upload = QtWidgets.QLabel("Upload") self.vBox_upload.addWidget(self.lbl_upload) self.lbl_upload_val = QtWidgets.QLabel(dpr.parse_date(video["Upload"])) self.vBox_upload.addWidget(self.lbl_upload_val) self.hBox_top.addLayout(self.vBox_upload) self.vBox_gathered = QtWidgets.QVBoxLayout() self.lbl_gathered = QtWidgets.QLabel("Gathered") self.vBox_gathered.addWidget(self.lbl_gathered) self.lbl_gathered_val = QtWidgets.QLabel(dpr.parse_date(video["Gathered"])) self.vBox_gathered.addWidget(self.lbl_gathered_val) self.hBox_top.addLayout(self.vBox_gathered) self.vBox_all.addLayout(self.hBox_top) self.vBox_all.addStretch() self.addLayout(self.vBox_all) self.addStretch()
def cmb_channel_timeframe(self, index): dpr.filter_channel_timeframe(self.cmb_channel_list_timeframe.itemText(index))
def __init__(self, channel): QHBoxLayout.__init__(self) self.id = channel.name self.chan_thumbdata = dpr.get_thumb_data(self.id, "Channel") self.img_chan = QtGui.QPixmap() self.img_chan.loadFromData(self.chan_thumbdata) self.img_chan = self.img_chan.scaledToHeight(80) self.lbl_chan = QtWidgets.QLabel() self.lbl_chan.mouseReleaseEvent = self.goToChannel self.lbl_chan.setPixmap(self.img_chan) self.addWidget(self.lbl_chan) self.vBox_all = QtWidgets.QVBoxLayout() self.vBox_all.addStretch() self.lbl_name = QtWidgets.QLabel(channel["Title"], font=f_subheader) self.vBox_all.addWidget(self.lbl_name) self.hBox_top = QtWidgets.QHBoxLayout() self.vBox_subscribers = QtWidgets.QVBoxLayout() self.lbl_subscribers = QtWidgets.QLabel("Subscribers") self.vBox_subscribers.addWidget(self.lbl_subscribers) self.lbl_subscribers_val = QtWidgets.QLabel(str(channel["Subscribers"])) self.vBox_subscribers.addWidget(self.lbl_subscribers_val) self.hBox_top.addLayout(self.vBox_subscribers) self.vBox_views = QtWidgets.QVBoxLayout() self.lbl_views = QtWidgets.QLabel("Views") self.vBox_views.addWidget(self.lbl_views) self.lbl_views_val = QtWidgets.QLabel(str(channel["Views"])) self.vBox_views.addWidget(self.lbl_views_val) self.hBox_top.addLayout(self.vBox_views) self.vBox_num_vids = QtWidgets.QVBoxLayout() self.lbl_num_vids = QtWidgets.QLabel("# Videos") self.vBox_num_vids.addWidget(self.lbl_num_vids) self.lbl_num_vids_val = QtWidgets.QLabel(str(len(dpr.get_scanned_videos(self.id)))) self.vBox_num_vids.addWidget(self.lbl_num_vids_val) self.hBox_top.addLayout(self.vBox_num_vids) self.vBox_comments = QtWidgets.QVBoxLayout() self.lbl_comments = QtWidgets.QLabel("Comments") self.vBox_comments.addWidget(self.lbl_comments) self.lbl_comments_val = QtWidgets.QLabel(str(channel["Comments"])) self.vBox_comments.addWidget(self.lbl_comments_val) self.hBox_top.addLayout(self.vBox_comments) self.vBox_upload = QtWidgets.QVBoxLayout() self.lbl_upload = QtWidgets.QLabel("Upload") self.vBox_upload.addWidget(self.lbl_upload) self.lbl_upload_val = QtWidgets.QLabel(dpr.parse_date(channel["Upload"])) self.vBox_upload.addWidget(self.lbl_upload_val) self.hBox_top.addLayout(self.vBox_upload) self.vBox_gathered = QtWidgets.QVBoxLayout() self.lbl_gathered = QtWidgets.QLabel("Gathered") self.vBox_gathered.addWidget(self.lbl_gathered) self.lbl_gathered_val = QtWidgets.QLabel(dpr.parse_date(channel["Gathered"])) self.vBox_gathered.addWidget(self.lbl_gathered_val) self.hBox_top.addLayout(self.vBox_gathered) self.vBox_all.addLayout(self.hBox_top) self.vBox_all.addStretch() self.addLayout(self.vBox_all) self.addStretch()
def show_tfr_plot(self): DataProcessing.generate_and_plot_waveletAnalysis(self.canvas.rawData,self.canvas.indices[0],self.canvas.startTime,self.canvas.endTime,self.canvas.samplingRate)
def setupAddAndScan(self): self.tab_add_scan = QtWidgets.QWidget() self.tab_add_scan.setEnabled(True) self.tab_add_scan.setAccessibleName("") self.tab_add_scan.setObjectName("tab_add_scan") self.hBox_add_scan_2 = QtWidgets.QHBoxLayout(self.tab_add_scan) self.hBox_add_scan_2.setObjectName("hBox_add_scan_2") self.vBox_add_scan_1 = QtWidgets.QVBoxLayout() self.vBox_add_scan_1.setObjectName("vBox_add_scan_1") self.hBox_add_scan_1 = QtWidgets.QHBoxLayout() self.hBox_add_scan_1.setObjectName("hBox_add_scan_1") self.line_add_url = QtWidgets.QLineEdit(self.tab_add_scan) self.line_add_url.setFixedSize(350,22) self.line_add_url.setObjectName("line_add_url") self.line_add_url.setText("Insert URL") self.hBox_add_scan_1.addWidget(self.line_add_url) self.btn_add_video = QtWidgets.QPushButton(self.tab_add_scan) self.btn_add_video.setToolTipDuration(1) self.btn_add_video.setObjectName("btn_add_video") self.btn_add_video.setText("Add Video") self.btn_add_video.setToolTip("Add a video to the database.") self.btn_add_video.clicked.connect(self.btn_add_video_press) self.hBox_add_scan_1.addWidget(self.btn_add_video) self.btn_add_channel = QtWidgets.QPushButton(self.tab_add_scan) self.btn_add_channel.setToolTipDuration(1) self.btn_add_channel.setObjectName("btn_add_channel") self.btn_add_channel.setText("Add Channel") self.btn_add_channel.setToolTip("Add a chanel to the database") self.btn_add_channel.clicked.connect(self.btn_add_channel_press) self.hBox_add_scan_1.addWidget(self.btn_add_channel) self.hBox_add_scan_1.addStretch() self.vBox_add_scan_1.addLayout(self.hBox_add_scan_1) self.grid = QtWidgets.QGridLayout() self.vBox_recent_videos = QtWidgets.QVBoxLayout() self.lbl_recent_videos = QtWidgets.QLabel("Last Uploaded Videos", font=f_header) self.vBox_recent_videos.addWidget(self.lbl_recent_videos) self.recent_videos = dpr.get_sorted(df_videos, "Upload", 3) self.recent_video_details = [videoDetails(self.recent_videos.iloc[x]) for x in range(len(self.recent_videos))] for i in range(len(self.recent_video_details)): self.vBox_recent_videos.addLayout(self.recent_video_details[i]) self.grid.addLayout(self.vBox_recent_videos, 0,0) self.vBox_add_scan_video_details = QtWidgets.QVBoxLayout() self.vBox_add_scan_video_details.addStretch() self.lbl_all_videos = QtWidgets.QLabel("Total Videos: " + str(dpr.get_num_items(df_videos)), font=f_preview) self.vBox_add_scan_video_details.addWidget(self.lbl_all_videos) self.lbl_all_video_views = QtWidgets.QLabel("Total Video Views: " + str(dpr.get_latest_sum(r"video_views.csv")), font=f_preview) self.vBox_add_scan_video_details.addWidget(self.lbl_all_video_views) self.lbl_all_likes_dislikes = QtWidgets.QLabel("Total Likes/Dislikes: " + str(dpr.get_latest_sum(r"video_likes.csv")) + "/" + str(dpr.get_latest_sum(r"video_dislikes.csv")), font=f_preview) self.vBox_add_scan_video_details.addWidget(self.lbl_all_likes_dislikes) self.lbl_dur_total = QtWidgets.QLabel("Runtime of all Videos combined: " + str(dpr.get_dur_total()), font=f_preview) self.vBox_add_scan_video_details.addStretch() self.hBox_stretch_videos = QtWidgets.QHBoxLayout() self.hBox_stretch_videos.addLayout(self.vBox_add_scan_video_details) self.hBox_stretch_videos.addStretch() self.grid.addLayout(self.hBox_stretch_videos, 0,1) self.vBox_recent_channels = QtWidgets.QVBoxLayout() self.lbl_recent_channels = QtWidgets.QLabel("Last Added Channels", font=f_header) self.vBox_recent_channels.addWidget(self.lbl_recent_channels) self.recent_channels = dpr.get_sorted(df_channels, "Gathered", 3) self.recent_channel_details = [channelDetails(self.recent_channels.iloc[x]) for x in range(len(self.recent_channels))] for i in range(len(self.recent_channel_details)): self.vBox_recent_channels.addLayout(self.recent_channel_details[i]) self.grid.addLayout(self.vBox_recent_channels, 1,0) self.vBox_add_scan_channel_details = QtWidgets.QVBoxLayout() self.vBox_add_scan_channel_details.addStretch() self.lbl_all_channels = QtWidgets.QLabel("Total Channels: " + str(dpr.get_num_items(df_channels)), font=f_preview) self.vBox_add_scan_channel_details.addWidget(self.lbl_all_channels) self.lbl_all_subscribers = QtWidgets.QLabel("Total Subscribers: " + str(dpr.get_latest_sum(r"channel_subscribers.csv")), font=f_preview) self.vBox_add_scan_channel_details.addWidget(self.lbl_all_subscribers) self.vBox_add_scan_channel_details.addStretch() self.grid.addLayout(self.vBox_add_scan_channel_details, 1,1) self.vBox_add_scan_1.addLayout(self.grid) spacerItem = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding) self.vBox_add_scan_1.addItem(spacerItem) self.hBox_add_scan_2.addLayout(self.vBox_add_scan_1)
#try to match the IDs of all videos or channels with the IDs of the data from gather_data. self.stats = {} for i in self.id_list: try: self.stats[i] = self.data[i][stat] except KeyError as e: self.stats[i] = "NaN" self.df_stats = pd.DataFrame.from_dict(self.stats, orient="index", columns=[self.curr_time]) #if there is no previous channels- or videos-file or if they are empty, don't merge with df_stats before writing try: self.df_old = dpr.get_dataframe(output_file) self.df_new = self.df_old.join(self.df_stats, how="outer") dpr.write_dataframetofile(self.df_new, output_file) except (ValueError, OSError): dpr.write_dataframetofile(self.df_stats, output_file) if __name__ == "__main__": channels = DataContainer("channels", "channels.csv") dpr.update_videos(channels.id_list) #Look for new Videos in all channels before updating the rest channels.gather_data() channels.update_stats("channel_views.csv", "viewCount") channels.update_stats("channel_subscribers.csv", "subscriberCount") channels.update_stats("channel_videos.csv", "videoCount") channels.update_stats("channel_comments.csv", "commentCount") videos = DataContainer("videos", "videos.csv") videos.gather_data() videos.update_stats("video_views.csv", "viewCount") videos.update_stats("video_likes.csv", "likeCount") videos.update_stats("video_dislikes.csv", "dislikeCount") videos.update_stats("video_comments.csv", "commentCount")
def cmb_video_timeframe(self, index): dpr.filter_video_timeframe(self.cmb_video_list_timeframe.itemText(index))
def cmb_channel_popularity(self, index): self.tbl_channel_list.populate(dpr.prep_table_dsp(dpr.sort_channels(self.cmb_channel_list_popularity.itemText(index), self.tbl_channel_list.df))) self.tbl_channel_list.df = dpr.sort_channels(self.cmb_channel_list_popularity.itemText(index), self.tbl_channel_list.df)
def confirm(self): if self.type == "Video": dpr.delete_video(self.id) elif self.type == "Channel": dpr.delete_channel(self.id) self.close()