def __init__(self):
Frame.__init__(self)
self.master.title("ASL Game")
self.master.minsize(width=500,height=500)
self.grid()
self.__dataPane = Frame(self)
self.__dataPane.grid(row=0, column=0)
self.__mode1Frame = Frame(self)
self.__mode2Frame = Frame(self)
## self.__mode3Frame = Frame(self)
self.__dp = DataProcessing()
self.__whole = self.__dp.getWholeSignDict()
self.__NumLtr = self.__dp.getNumLtrDict()
self.__homeButton = Button(self.__dataPane, text="Home",\
command=self.__home)
self.__welcome = Label(self.__dataPane, text="Choose a mode")
self.__mode1Button = Button(self.__dataPane, text="Mode 1",\
command=self.__mode1)
self.__mode2Button = Button(self.__dataPane, text="Mode 2",\
command=self.__mode2)
## self.__mode3Button = Button(self.__dataPane, text="Mode 3",\
## command=self.__mode3)
self.__homeButton.grid(row=1, column=1)
self.__welcome.grid(row=2, column=2)
self.__mode1Button.grid(row=3, column=1)
self.__mode2Button.grid(row=3, column=2)
#self.__mode3Button.grid(row=3, column=3)
mainloop()
def plotByTTM(df, t, dim, subject_list, ylabel):
k = dim[0]*dim[1]
split_moneyness = DataProcessing.weightedSplit(df, k, 'moneyness')
split_ttm = []
for moneyness in split_moneyness:
split_ttm.append(DataProcessing.weightedSplit(moneyness, t, 'ttm'))
fig = plt.figure()
for subject in subject_list:
plot_list = []
for moneyness in split_ttm:
temp = []
for m in moneyness:
temp.append([m['ttm'].median(), m[subject].mean()])
plot_list.append(np.array(temp))
count = 1
subplots = []
height = dim[1]
for i in range(len(plot_list)):
subplots.append(fig.add_subplot(int(str(height)+str(dim[0])+str(count))))
count += 1
for j in range(len(subplots)):
subplots[j].plot(plot_list[j][:, 0], plot_list[j][:, 1])
subplots[j].set_title('Moneyness: '+str(round(split_moneyness[j]['moneyness'].min(), 3))+' - '+str(round(split_moneyness[j]['moneyness'].max(), 3)))
subplots[j].set_xlabel('time-to-maturity')
subplots[j].set_ylabel(ylabel)
plt.show()
def Svmker():
x_train, x_test, y_train, y_test = train_test_split(dp.dataSet(2),
dp.label(2),
test_size=0.1,
random_state=1)
grid = [{'kernel': ['rbf'], 'gamma': [1e-2, 1e-4], 'C': [1, 10, 100]}]
grid = GridSearchCV(SVC(), grid, cv=5)
grid.fit(x_train, y_train)
print("The best parameters are %s with a score of %0.2f" %
(grid.best_params_, grid.best_score_))
def plotByMoneyness(df, k, dim, subject_list, ylabel):
t = dim[0]*dim[1]
split_ttm = DataProcessing.weightedSplit(df, t, 'ttm')
split_moneyness = []
for ttm in split_ttm:
split_moneyness.append(DataProcessing.weightedSplit(ttm, k, 'moneyness'))
fig = plt.figure()
subplots = []
plot_list = []
axins_list = []
for subject in subject_list:
for ttm in split_moneyness:
temp = []
for m in ttm:
temp.append([m['moneyness'].median(), m[subject].mean()])
plot_list.append(np.array(temp))
height = dim[1]
for i in range(t):
subplots.append(fig.add_subplot(int(str(height)+str(dim[0])+str(i+1))))
# axins_list.append(zoomed_inset_axes(subplots[i], 2, loc=1))
subplots[i].set_title('Maturity: '+str(int(split_ttm[i]['ttm'].min()*365))+' - '+str(int(split_ttm[i]['ttm'].max()*365))+' days')
subplots[i].set_xlabel('moneyness')
subplots[i].set_ylabel(ylabel)
count = 0
for j in range(len(subject_list)):
# xmin = 1.025
for l in range(len(subplots)):
subplots[l].plot(plot_list[count][:, 0], plot_list[count][:, 1], label=str(subject_list[j]))
# axins_list[l].plot(plot_list[count][:, 0], plot_list[count][:, 1])
# axins_list[l].set_xlim(xmin, plot_list[count][:, 0].max()+0.01) # apply the x-limits
# axins_list[l].set_ylim(plot_list[count][:, 1].min()-0.01, plot_list[count][:, 1][-1]+0.02) # apply the y-limits
# axins_list[l].xaxis.set_visible(False)
# axins_list[l].yaxis.set_visible(False)
count += 1
# for p in range(len(axins_list)):
# mark_inset(subplots[p], axins_list[p], loc1=3, loc2=1, fc="none", ec="0.5")
#
# axins_list[2].set_xlim(1., 1.1) # apply the x-limits
# axins_list[3].set_xlim(1., 1.1) # apply the x-limits
# axins_list[4].set_xlim(1., 1.1) # apply the x-limits
# axins_list[5].set_xlim(1., 1.1) # apply the x-limits
# axins_list[6].set_xlim(1., 1.1) # apply the x-limits
# axins_list[7].set_xlim(1., 1.1) # apply the x-limits
# axins_list[8].set_xlim(1., 1.2) # apply the x-limits
plt.show()
def expected_time_to_trigger(bd, outdir):
"""
Represents the "hardness" of each triggered bug in a heatmap
by computing the expected time to trigger for each bug
:param bd: { A BenchmarkData object loaded from experiment summary file }
:type bd: { BenchmarkData }
"""
ett, agg = DataProcessing.expected_time_to_trigger_data(bd)
#Compute the order of the fuzzer that found the most bugs in descending order
fuzzer_order = DataProcessing.number_of_unique_bugs_found_data(bd)
fuzzer_order = fuzzer_order.sort_values(by=['Bugs'],ascending = False) \
.reset_index()['Fuzzer'].tolist()
#Sort the bug by aggregate time
ett['Aggregate'] = agg
ett.sort_values(by='Aggregate', inplace=True)
ett = ett.drop(labels='Aggregate', axis=1)
#Reordering the fuzzers
ett = ett[fuzzer_order]
fuzzer_label = list(ett.columns)
bug_label = list(ett.index)
annotations = ett.copy()
annotations[fuzzer_label] = annotations[fuzzer_label].applymap(
lambda x: pp_time(x))
fig, ax = plt.subplots(figsize=(10, 10))
plt.yticks(rotation=0)
#Norm factor has to been precomputed
heat_map = sns.heatmap(np.array(ett),
cmap='seismic',
xticklabels=fuzzer_label,
yticklabels=bug_label,
annot=np.array(annotations),
fmt='s',
norm=colors.PowerNorm(gamma=0.32),
ax=ax)
#Color bar properties
max_num_trials = bd.frame.reset_index().groupby('Fuzzer')['Campaign'] \
.nunique().max()
xticks = list_ticks(bd.duration * max_num_trials)[4:]
xticklables = list(map(lambda x: pp_time(x), xticks))
cbar = ax.collections[0].colorbar
cbar.set_ticks(xticks)
cbar.set_ticklabels(xticklables)
ax.patch.set(fill='True', color='darkgrey')
ax.xaxis.tick_top()
name, path = output(outdir, 'data', 'summary_expected_ttb_heat.svg')
fig.savefig(path)
plt.close()
return path
def main():
# 下载数据.
raw_data = DataProcessing.download_data()
# 数据编码及处理.
data = DataProcessing.onehot_data(raw_data)
data = data.values.astype(np.float32)
# 将数据顺序打乱并将数据以8:2分为训练数据与测试数据.
np.random.shuffle(data)
scale = int(0.8 * len(data))
train_data = data[:scale]
test_data = data[scale:]
INPUT = tf.placeholder(tf.float32, [None, 25], "input")
backward(train_data)
def confirm(self):
dpr.add_item(self.data)
if self.data[0]["Type"] == "Channel":
dpr.update_videos(self.data[0]["ID"])
elif self.data[0]["Type"] == "Video":
dpr.channel_from_video(self.data[0]["ID"])
dpr.refresh()
self.close()
def test():
img = cv2.imread("./img/153g.bmp", 0)
ip = ImageProcessing.ImageProcessing(img)
ip.doHoughTrans()
img = ip.cannyImg[ip.C - 125:ip.C + 125, :]
waveHeight, waveLenght = dp.getWaveParaByPeak(img)
def train():
"""
训练模型
:return:
"""
du = DataProcessing.DataUnit(**data_config)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
steps = int(len(du) / batch_size) + 1
# 创建session的时候设置显存根据需要动态申请
tf.reset_default_graph()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
# 定义模型
model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=du.vocab_size,
decoder_vocab_size=du.vocab_size,
mode='train',
**model_config)
init = tf.global_variables_initializer()
writer = tf.summary.FileWriter('./graph/nlp', sess.graph)
sess.run(init)
if continue_train:
model.load(sess, save_path)
model.export(sess)
'''
def chatbot_api(infos):
du = DataProcessing.DataUnit(**data_config)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
batch_size = 1
tf.reset_default_graph()
model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=du.vocab_size,
decoder_vocab_size=du.vocab_size,
mode='decode',
**model_config)
# 创建session的时候允许显存增长
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
init = tf.global_variables_initializer()
sess.run(init)
model.load(sess, save_path)
while True:
q = infos
if q is None or q.strip() == '':
return "请输入聊天信息"
continue
q = q.strip()
indexs = du.transform_sentence(q)
x = np.asarray(indexs).reshape((1, -1))
xl = np.asarray(len(indexs)).reshape((1, ))
pred = model.predict(sess, np.array(x), np.array(xl))
result = du.transform_indexs(pred[0])
return result
def runSpikeDetection(self):
"""runs stupid spike detector on only 1st channel"""
channelIndex = self.canvas.indices[0]
self.rawCh1Spikes = DataProcessing.stupidIdentifySpikes(self.canvas.rawData[channelIndex,:][0],cutoff=float(self.thresholdEdit.text()))
self.ch1Spikes = self.canvas.rawData.index_as_time(self.rawCh1Spikes)
self.canvas.annotationTimes = self.ch1Spikes[0]
QtGui.QMessageBox.information(None,"Report","{d} spikes found.\n\n{i}".format(d=len(self.ch1Spikes[0]),i=self.ch1Spikes[0]))
def simulation(self, simulate, selected_nodes_R):
"""
The function to perform a simulation rollout (Delete a sequence of symbols).
:param simulate: (ELLS-node) The start node for simulation.
:param selected_nodes_R: (float) The total R-score that 'simulate' node already get.
:return: (float) The total R-score of simulation.
"""
state = simulate.state
actions = len(state) + self.delta - len(self.Z)
simulation_reward = selected_nodes_R + self.ELLS_Total
for i in range(actions):
while 1:
policy = random.randint(0, len(state) - 1)
# The legal deleted characters cannot be the same as the neighbours. Such
# method can ensure that the symbols deleted for this simulation node are
# completely different each time.
# Experimental result proves that random function is more effective than
# searching for all legal deletions for each deletions.
neighbor_right = policy + 1
if neighbor_right < len(state):
if state[policy] == state[neighbor_right]:
continue
# Consider only the deletion that does not incur any sensitive patterns.
reinstate = False
after = state[_non_zero(policy - self.k + 1):policy] + state[policy + 1: policy + self.k]
for j in range(len(after) - self.k + 1):
if self._is_sensitve(after[j:j + self.k]):
reinstate = True
break
if not reinstate: break
simulation_reward += self._extract_r_score(state, policy)
state = DataProcessing.delete(state, policy)
return simulation_reward
def index(self):
indoor_temp, outdoor_temp = DataProcessing.generate_graph()
with open("/home/pi/Developpement/pyled/index.html") as file:
html = ''
for l in file:
html += l
return html.format(indoor_temp, outdoor_temp)
def onAmplitudeChanged(self, nAmplitude):
self.storedAmplitude = nAmplitude;
if(self.rawData):
self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass,self.indices)
self.signalData = np.float32(10000*np.array(self.displayData[0]))
self.program['a_position'] = self.signalData.reshape(-1, 1)
self.update()
def refresh(self):
indoorTemp, outdoorTemp = DataProcessing.generate_complete_data()
with open("/home/pi/Developpement/pyled/current_temp.txt",
'w') as file:
file.write(str(indoorTemp) + " " + str(outdoorTemp))
print(indoorTemp, outdoorTemp)
raise cherrypy.HTTPRedirect("/")
def data_preprocess(self):
# 正运动学求取末端位姿
X = np.zeros([self.num, 6])
for i in range(self.num):
X[i, :] = kin.fkine_euler(self.DH_0, self.qq_array[i, :])
#求取末端位置的,前向差分获取速度和加速度
X_xva = np.zeros([self.num, self.m, 3])
X_xva[:, :, 0] = X
#建立高增益观测器
observer = dp.HighGainObserver()
for i in range(self.m):
observer.get_original_data(X[:, i], self.T)
X_xva[:, i, 1] = observer.put_observer_data()
observer.get_original_data(X_xva[:, i, 1], self.T)
X_xva[:, i, 2] = observer.put_observer_data()
# for i in range(self.num):
# X_xva[i, :, 0] = X[i, :]
# if (i==0):
# X_xva[i, :, 1] = 0
# X_xva[i, :, 2] = 0
# else:
# X_xva[i, :, 1] = (X_xva[i, :, 0] - X_xva[i-1, :, 0])/self.T
# X_xva[i, :, 2] = (X_xva[i, :, 1] - X_xva[i-1, :, 1])/self.T
self.X_xva = X_xva
def _init_funcs(self,
number=None,
blacklist_defaults=True,
blacklist_forms=True,
csvdata=None):
'''initial functions, once self.files is set.'''
if number is not None: self.number = number
else:
if csvdata is not None:
import DataProcessing as dp
self.number = dp.poke_to_N(self.pokemon, csvdata)
else:
self.number = None
if self.number is not None: self.number = str(self.number).zfill(3)
took_pre = [take_pre(s) for s in self.files]
self.pre = [x[0] for x in took_pre]
self.sites = [x[1] for x in took_pre]
took_ext = [take_ext(s) for s in self.sites]
self.ext = [x[1] for x in took_ext]
self.names = [x[0] for x in took_ext]
self.comps = [x.split('_') for x in self.names]
self._idx = np.arange(len(self.files))
self.idx = np.arange(len(self.files))
if blacklist_defaults:
self.blacklist_defaults()
if blacklist_forms:
self.blacklist_forms()
def __init__(self, keyname, training_data):
"""
Initialize the TrainingSet class.
This class contains Session objects, which represent single sessions of training
data on a user-by-user basis. Session objects are passed with their member DataPoints
to the DataProcessing module where they can be analyzed and manipulated.
:keyname represents the name of the indepdendent variable for which we will try to
make predictions.
:training_data represents the file contents of the saved session
files, passed to the initializer as a list of file contents.
"""
# Create a new featureset object and pass it the name of the prediction target variable
self.featureset = Featureset(keyname)
self.classify_on_key = keyname
self.training_data = training_data
# Kick off the training session by actually creating session objects in self.sessions
if self.parse() is True:
# Now create Data Processing objects and pass all of the sessions and features
self.processor = DataProcessing.DataProcessor(
self.featureset, self.sessions)
# Do more things with the DataProcessor...
else:
sys.stderr.write("Error parsing data: exiting...")
def draw_bbox_video(image, data, g_cerca, g_lejos):
color_red = (0, 0, 255)
color_green = (0, 255, 0)
ind_p, p_bbox_center, pair_point_bbox = datap.process_df2bbox(data)
for pair_points in g_cerca:
points_line = []
for points in pair_points:
id_people = points[2]
ind_list = ind_p.index(id_people)
cv2.rectangle(img=image,
pt1=point_float2int(pair_point_bbox[ind_list][0]),
pt2=point_float2int(pair_point_bbox[ind_list][1]),
color=color_red,
thickness=2)
points_line.append(p_bbox_center[ind_list])
cv2.line(image, points_line[0], points_line[1], color_red, thickness=2)
for Po in g_lejos:
id_people = Po[2]
ind_list = ind_p.index(id_people)
cv2.rectangle(img=image,
pt1=point_float2int(pair_point_bbox[ind_list][0]),
pt2=point_float2int(pair_point_bbox[ind_list][1]),
color=color_green,
thickness=2)
return image
def baseline(self):
"""
The function of the baseline algorithm. Firstly, sort the distortion of each pattern in descending
order. Second, traverse each letter from the most distorted pattern to the least distorted pattern.
The loop will terminate if and only if the current candidate letter does not incur any sensitive pattern.
:return BA: (String) the string type of data, whose length is |Z| - delta.
"""
Z = self.Z
w_nsens = self.non_sens_w
z_nsens = self._search_nsens(Z)
for i in range(self.delta):
# A list of all candidate patterns' distortions in Z
candidates = [(abs(z_nsens[Z[j:j + self.k]] - w_nsens[Z[j:j + self.k]]), list(range(j, j + self.k)))
for j in range(len(Z) - self.k + 1)]
# Sort distortions in descending order.
# The most distorted pattern will be considered first.
candidates.sort(key=lambda x: x[0], reverse=True)
for candidate in candidates:
for victim in candidate[1]:
after = Z[_non_zero(victim - self.k + 1):victim] + Z[victim + 1: victim + self.k]
non_sens_after = self._search_nsens(after)
# check if 'victim' led to any sensitive patterns.
if len(set(non_sens_after.keys()).intersection(set(self.S))) == 0:
Z = DataProcessing.delete(Z, victim)
z_nsens = self._search_nsens(Z)
break
else:
continue
break
else:
continue
break
return Z
def TrimData(data: pd.DataFrame,
linearization_point: LinearizationPoint,
data_constraints: dict = None):
"""
dataConstraints is a map of data key to maximum allowable deviation
"""
if data_constraints is not None:
if type(linearization_point) is LinearizationPoint:
lp_dict = linearization_point._asdict()
else:
lp_dict = linearization_point.__dict__
data = DataProcessing.constrain_data(data, lp_dict, data_constraints)
v = data['v'] - linearization_point.v
p = data['p']
r = data['r']
phi = data['phi'] - linearization_point.phi
vdot = data['v_dot']
pdot = data['p_dot']
rdot = data['r_dot']
aileron = data['roll_ctrl'] - linearization_point.aileron_ctrl
rudder = data['yaw_ctrl'] - linearization_point.rudder_ctrl
return TrimmedData(v, p, r, phi, vdot, pdot, rdot, aileron, rudder)
def learn_boosted(tdepth, nummodels, datapath):
train_location = datapath + '/agaricuslepiotatrain1.csv'
f = open(train_location, 'r')
data = DataProcessing.preprocess(f)
test_location = datapath + '/agaricuslepiotatest1.csv'
t = open(test_location, 'r')
tdata = DataProcessing.preprocess(t)
boosting_models = boosting(data, nummodels, tdepth)
accuracy = classify_accu(boosting_models, tdata)
print("Accuracy : ", accuracy)
def onTextBoxesChanged(self, lowPass, highPass):
self.lowPass = lowPass;
self.highPass = highPass;
print "startTime: {s}, endTime: {e}, lowPass: {l}, highPass: {h}".format(s=self.startTime, e=self.endTime, l=self.lowPass, h=self.highPass)
self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass,self.indices)
self.signalData = np.float32(10000*np.array(self.displayData[0]))
self.program['a_position'] = self.signalData.reshape(-1, 1)
self.update()
def SaveDataToFile():
try:
objF = DataProcessing.DataProcessing()
objF.FileName = "CustomerData.txt"
objF.TextData = Customers.CustomerList.ToString()
objF.SaveData()
except Exception as e:
print(e)
def __init__(self, type, input_file):
filename = dpr.get_datafilepath(input_file)
#create a list with IDs of all channels or all videos, so we can loop through them later
with open(filename, "r") as f:
self.reader = csv.reader(f, delimiter="\t")
self.id_list = [rows[0] for rows in self.reader]
self.id_list.pop(0) #remove header row
self.type = type #save instance as either channel or video object
def Operate(Specs):
dataObj = DataProcessing.DataProcessing(Specs)
# TBD make sure that input data is valid etc.
# TBD back test results against real data
# TBD Error handling
# TBD fix model paths
#print("----------------------------------------------------------")
#print("event forecast started.")
#print(datetime.now())
#print("----------------------------------------------------------")
if not hasattr(dataObj, "data"):
print("no valid data could be extracted.")
return
if Specs.Mode == "2":
print("No Neural Network support at the moment.")
return
if not int(Specs.AnzahlPrognosen) == 1 or not Specs.AnzahlPrognosen:
print(
"corrected wrong configuration: Only realtime forecast is considered"
)
Specs.AnzahlPrognosen = 1
print("input data: ", dataObj.datapath)
#load model
try:
mdlObj = generateModel(Specs, dataObj)
except Exception as e:
print("problems while loading model: " + e)
try:
ForecastDyn = mdlObj.predictOperative(Specs.horizont)
timeseriesNF = dataObj.data["NoFilter"]
except Exception as e:
print("problems while prediction:\n" + e)
try:
print("creating csv.")
createCsv(ForecastDyn, Specs)
print("done")
except Exception as e:
print("no csv could be created:\n" + e)
try:
DataDyn = timeseriesNF[-Specs.horizont:]
multi = "\n##### " + str(
Specs.horizont) + "-step Prediction against last values ##### \n"
print(multi)
sys.stdout.flush()
FDyn = ErrorAnalysis.ErrorAnalysis(DataDyn, ForecastDyn)
print(FDyn.criterias)
sys.stdout.flush()
except Exception as e:
print(e)
def _ELLS_ALGO(self, root, iterations):
"""
The main function of ELLS-ALGO. This function recursively finding the most promising symbol to delete
by running a certain number of Monte Carlo cycles (Selection -> Expansion -> Simulation
-> Backpropagation) until all optimal delete characters.
:param root: (ELLS_node) The root node of the tree stores the string to be deleted.
:param iterations: (int) The number of deletions required to make a decision(select deleted symbol)
:return: H:(string) The resulting data after deletions.
"""
# Base case of recursion: recursion terminates as only one remaining symbol need to be deleted.
if self.remain_delete == 1:
score = self._get_legal_deletions(root.state, strategy='all') # greedy find current best deleted symbol
self.ELLS_Total += score[0]
result = DataProcessing.delete(root.state, score[1])
self.ELLS_track.append(result)
return result
# Expand the root node beforehand.
if not root.isExpanded:
root.expand(self._get_legal_deletions(root.state), self.C)
for _ in range(iterations):
# Selection
simulate, selected_nodes_R = root.select_leaf()
# Expansion delay
if simulate.visits > 3 and len(self.Z) - len(simulate.state) < self.delta:
# Expansion
simulate.expand(self._get_legal_deletions(simulate.state), self.C)
simulate = simulate.children[next(iter(simulate.children))]
simulate_score = selected_nodes_R + simulate.score
else:
simulate_score = selected_nodes_R
# Simulation
simulation_reward = self.simulation(simulate, simulate_score)
# Backpropagation
simulate.backpropagation(simulation_reward)
# Final deleted symbol selection -> select the node by max-robust child selection policy.
best_child = None
best_result = - float('inf')
for child in root.children.values():
score = - child.reward + child.visits
if score > best_result:
best_result = score
best_child = child
self.ELLS_track.append(best_child.state)
# Search tree reuse
iterations = self.max_simulations - np.sum([child.visits for child in best_child.children.values()])
self.remain_delete -= 1
self.ELLS_Total += best_child.score
best_child.refresh()
gc.collect()
# transform best root child to next tree's root node.
return self._ELLS_ALGO(best_child, int(iterations))
def buttonClicked(self):
sid = int(self.ui.comboBox.currentText().split("-")[0].strip())
year = int(self.ui.comboBox_2.currentText())
season = int(self.ui.comboBox_3.currentText())
self.df = DataProcessing.crwal_financial_report(sid, year, season)
model = PandasModel(self.df)
self.ui.tableView.setModel(model)
def update_stats(self, output_file, stat):
#create df_stats with only relevant data(1 column - date = name)
self.curr_time = time.strftime("%Y-%m-%dT%H:%M:%S.000Z")
#try to match the IDs of all videos or channels with the IDs of the data from gather_data.
self.stats = {}
for i in self.id_list:
try:
self.stats[i] = self.data[i][stat]
except KeyError as e:
self.stats[i] = "NaN"
self.df_stats = pd.DataFrame.from_dict(self.stats, orient="index", columns=[self.curr_time])
#if there is no previous channels- or videos-file or if they are empty, don't merge with df_stats before writing
try:
self.df_old = dpr.get_dataframe(output_file)
self.df_new = self.df_old.join(self.df_stats, how="outer")
dpr.write_dataframetofile(self.df_new, output_file)
except (ValueError, OSError):
dpr.write_dataframetofile(self.df_stats, output_file)
def SaveDatatoFile():
try:
ObjFile = DataProcessing.File()
ObjFile.TextFile = "CustomerDetails.txt"
ObjFile.TextData = Customers.CustomerList.ToString()
print("it worked, Done!")
ObjFile.SaveData()
except Exception as e:
print(e)
def trainTwoboxModel(modelName, model, epochs, modelPath="SavedModels"):
trainX, trainY = dp.dataPreprocessing(training_twobox)
testX, testY = dp.dataPreprocessing(testing_twobox)
modelPath += "twobox_" + modelName
if not os.path.exists(modelPath):
os.mkdir(modelPath)
model.fit(trainX,
trainY,
n_epoch=epochs,
shuffle=True,
validation_set=(testX, testY),
show_metric=True,
batch_size=BATCH_SIZE,
run_id=modelName + "_twobox")
model.save(modelPath + '/' + modelName + '.tfl')
def SaveDataToFile():
try:
objF = DataProcessing.DataProcessing()
objF.FileName = "EmployeeData.txt"
objF.TextData = Employees.EmployeeList.ToString()
print("Reached here")
objF.SaveData()
except Exception as e:
print(e)
def onStartEndChanged(self, startTime, endTime):
self.startTime = startTime;
self.endTime = endTime;
self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass,self.indices)
self.setupZoom(self.displayData)
# self.program.delete()
# self.program = gloo.Program(SERIES_VERT_SHADER, SERIES_FRAG_SHADER)
self.setupZoomStep2()
self.updateLines()
self.update()
def onAmplitudeChanged(self, nAmplitude):
self.storedAmplitude = nAmplitude
if (self.rawData):
self.displayData = DataProcessing.getDisplayData(
self.rawData, self.startTime, self.endTime,
self.storedAmplitude, self.lowPass, self.highPass,
self.indices)
self.signalData = np.float32(10000 * np.array(self.displayData[0]))
self.program['a_position'] = self.signalData.reshape(-1, 1)
self.update()
def test_2_getXY(self):
XY = DataProcessing.getXY(MERGEDA, "HC_AD")
self.assertEqual(np.array_equal(XY[1], [1,1,1,0,0,0]), True)
self.assertEqual(np.array_equal(XY[1], [1,1,1,1,0,0]), False)
XY = DataProcessing.getXY(MERGEDA, "MCI_AD")
self.assertEqual(np.array_equal(XY[1], [1,1,1,0,0,0,0]), True)
self.assertEqual(np.array_equal(XY[1], [1,1,1,1,0,0,0]), False)
XY = DataProcessing.getXY(MERGEDA, "HC_MCI")
self.assertEqual(np.array_equal(XY[1], [0,0,0,1,1,1,1]), True)
self.assertEqual(np.array_equal(XY[1], [0,0,0,0,1,1,1]), False)
self.assertEqual(np.array_equal(XY[2], ["RightAmygdala", "RightHippocampus"]), True)
self.assertEqual(np.array_equal(XY[2], ["RightHippocampus", "RightAmygdala"]), False)
self.assertEqual(np.array_equal(XY[4], MERGEDA), False)
print("getXY with MERGED A tested")
def __init__(self, df):
super(TableFromDf, self).__init__()
self.df = df
self.df_prep = dpr.prep_table_dsp(df)
self.setColumnCount(len(self.df_prep.columns))
self.setRowCount(len(self.df_prep.index))
self.setShowGrid(False)
self.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows)
self.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
self.populate(self.df_prep)
def train():
"""
训练模型
:return:
"""
du = DataProcessing.DataUnit(**data_config)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
trainset, _, _, _ = pickle.load(open('dataset.pkl', 'rb'))
steps = int(len(list(trainset)) / batch_size) + 1
print(trainset.shape)
# 创建session的时候设置显存根据需要动态申请
tf.reset_default_graph()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
# 定义模型
model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=du.vocab_size,
decoder_vocab_size=du.vocab_size,
mode='train',
**model_config)
init = tf.global_variables_initializer()
sess.run(init)
if continue_train:
model.load(sess, save_path)
loss_list = []
for epoch in range(1, n_epoch + 1):
costs = []
bar = tqdm(range(steps),
total=steps,
desc='epoch {}, loss=0.000000'.format(epoch))
for time in bar:
x, xl, y, yl = du.next_batch(batch_size, list(trainset))
max_len = np.max(yl)
y = y[:, 0:max_len]
cost, lr = model.train(sess, x, xl, y, yl, keep_prob)
costs.append(cost)
if epoch == 1 and time == 0:
loss_list.append(np.mean(costs))
if time == steps - 1:
loss_list.append(np.mean(costs))
bar.set_description(
'epoch {} loss={:.6f} lr={:.6f}'.format(
epoch, np.mean(costs), lr))
epoch_list = list(range(0, n_epoch + 1))
plt.xlabel('Epoch Number')
plt.ylabel('Loss Value')
plt.plot(epoch_list, loss_list)
plt.show()
model.save(sess, save_path=save_path)
def main(): # TODO preprocess the input file to get standard vectors configuration = config.get_config() filepath = configuration['datafile_path'] processed_data = DataProcessing.ProcessData(filepath) """ Model designing part """ # TODO design encoder for epi in range(configuration['max_epochs']): # print "training epoch ", epi # err = 0.0 num_steps = 0 # TODO: shuffle the training data and train this epoch ## train_start = time.time() # seq_lang_numpy = [] seq_world_numpy = [] seq_action_numpy = [] for name_map in configuration['maps_train']: max_steps = len( processed_data.dict_data['train'][name_map] ) print 'max_steps=', max_steps for idx_data, data in enumerate(processed_data.dict_data['train'][name_map]): # seq_lang_numpy, seq_world_numpy and seq_action_numpy will be set seq_lang_numpy, seq_world_numpy, seq_action_numpy = processed_data.process_one_data(idx_data, name_map, 'train') # np.concatenate((seq_lang_numpy, seq_lang)) # np.concatenate((seq_world_numpy, seq_world)) # np.concatenate((seq_action_numpy, seq_action)) """ trainer = Instantiates the model """ model = models.SeqToSeq() cost_numpy = model.build_model( seq_lang_numpy, # list of word indices seq_world_numpy, # matrix of dim (len(one_data['cleanpath'])*78 seq_action_numpy # index value of 1 in one hot vector of action ) print "Cost!!------", cost_numpy print "type = ", type(cost_numpy) print "shape = ", cost_numpy.shape print "---Cost_numpy___=",cost_numpy err += cost_numpy if idx_data % 100 == 99: print "training i-th out of N in map : ", (idx_data, max_steps, name_map) # num_steps += max_steps # train_err = err / num_steps
def main(): import DataProcessing as dp # This is my customized module for data pre-process, can be found here : https://github.com/newtoncircus/DEXTRA_SourceCodes/blob/master/DataProcessing.py root_path = "./" # path_predicted = "SampleSubmission_percent10.csv" # path_actual = "public_validation_datasheet_percent10.csv" path_predicted = "predicted.csv" path_actual = "true.csv" viki = dp.rakuten() (header_sample, arr_predicted) = viki.csv_to_array(root_path + path_predicted, "rU") (header_public, arr_actual) = viki.csv_to_array(root_path + path_actual, "rU") num = 3 EWAP(arr_predicted, arr_actual, num)
def btn_add_video_press(self):
self.video_data = dpr.check_video(self.line_add_url.text())
if type(self.video_data) is str:
self.video_add_popup = errorDialog("Error")
self.video_add_popup.display_text.setText(self.video_data)
self.video_add_popup.exec_()
elif type(self.video_data) is list:
self.video_add_popup = confirmDialog("Add Video", self.line_add_url.text(), self.video_data)
self.video_add_popup.display_text.setText(
"Is \"" + self.video_data[0]["Title"] + "\" your video?")
self.video_add_popup.exec_()
else:
print("wrong return format")
def handle_scroll(self, fdx):
dx = -fdx#-np.sign(event.delta[1])
if((self.startTime + dx) > 0.5):
self.startTime += dx
self.endTime += dx
elif ((self.startTime+dx) >= 0.0): #CB handle edge cases as we get close to beginning of dataset
self.startTime += dx
self.endTime += dx
olen=len(self.displayData[0][0])
self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass,self.indices)
if(len(self.displayData[0][0]) != olen):
self.onStartEndChanged(self.startTime,self.endTime)
self.signalData = np.float32(10000*np.array(self.displayData[0]))
self.program['a_position'] = self.signalData.reshape(-1, 1)
self.updateTextBoxes()
self.updateLines()
self.update()
def __init__(self, channel):
QWidget.__init__(self)
self.id = channel.name
self.setWindowIcon(QtGui.QIcon('app.png'))
self.setWindowTitle("Channel Details")
self.vBox_1 = QtWidgets.QVBoxLayout(self)
self.hBox_top = QtWidgets.QHBoxLayout()
self.lbl_header = QtWidgets.QLabel(channel["Title"], font=f_header)
self.hBox_top.addWidget(self.lbl_header)
self.hBox_top.addStretch()
self.btn_delete = QtWidgets.QPushButton("Delete Channel", self)
self.btn_delete.clicked.connect(self.delete_channel)
self.hBox_top.addWidget(self.btn_delete)
self.vBox_1.addLayout(self.hBox_top)
self.hBox_graphs = QtWidgets.QHBoxLayout()
if dpr.datafile_exists("channel_subscribers.csv") and dpr.datafile_exists("channels.csv"):
self.gra_channel_subs = LineGraphCanvas(
dpr.get_subs_ot(self.id),"Subscribers over Time")
self.hBox_graphs.addWidget(self.gra_channel_subs)
self.gra_channel_subs_growth = LineGraphCanvas(
dpr.get_growth(dpr.get_subs_ot(self.id)),"Subscriber Growth over Time")
self.hBox_graphs.addWidget(self.gra_channel_subs_growth)
self.vBox_1.addLayout(self.hBox_graphs)
self.hBox_details = QtWidgets.QHBoxLayout()
self.channel_details = channelDetails(channel)
self.hBox_details.addLayout(self.channel_details)
self.vBox_video_details = QtWidgets.QVBoxLayout()
self.lbl_video_header = QtWidgets.QLabel("Last Uploads", font=f_header)
self.vBox_video_details.addWidget(self.lbl_video_header)
self.videos = dpr.get_scanned_videos(self.id, df_videos)
self.recent_vids = dpr.get_sorted(self.videos, "Upload", 3)
self.video_details = [videoDetails(self.recent_vids.iloc[x]) for x in range(len(self.recent_vids))]
for i in range(len(self.video_details)):
self.vBox_video_details.addLayout(self.video_details[i])
self.hBox_details.addLayout(self.vBox_video_details)
self.hBox_details.addStretch()
self.vBox_1.addLayout(self.hBox_details)
self.vBox_1.addStretch()
def __init__(self, video):
QWidget.__init__(self)
print(video.name)
self.id = video.name
self.channel_id = dpr.get_channel_id(self.id)
self.setWindowIcon(QtGui.QIcon('app.png'))
self.setWindowTitle("Video Details")
self.vBox_1 = QtWidgets.QVBoxLayout(self)
self.hBox_top = QtWidgets.QHBoxLayout()
self.lbl_header = QtWidgets.QLabel(video["Title"], font=f_header)
self.hBox_top.addWidget(self.lbl_header)
self.hBox_top.addStretch()
self.btn_delete = QtWidgets.QPushButton("Delete Video", self)
self.btn_delete.clicked.connect(self.delete_video)
self.hBox_top.addWidget(self.btn_delete)
self.vBox_1.addLayout(self.hBox_top)
self.hBox_graphs = QtWidgets.QHBoxLayout()
if dpr.datafile_exists("video_views.csv") and dpr.datafile_exists("videos.csv"):
self.gra_video_views = LineGraphCanvas(
dpr.get_views_ot(self.id),"Views over Time")
self.hBox_graphs.addWidget(self.gra_video_views)
self.gra_video_views_growth = LineGraphCanvas(
dpr.get_growth(dpr.get_views_ot(self.id)),"Growth of Views over Time")
self.hBox_graphs.addWidget(self.gra_video_views_growth)
self.vBox_1.addLayout(self.hBox_graphs)
self.hBox_details = QtWidgets.QHBoxLayout()
self.video_details = videoDetails(video)
self.hBox_details.addLayout(self.video_details)
self.channel_details = channelDetails(df_channels.loc[self.channel_id])
self.hBox_details.addLayout(self.channel_details)
self.hBox_details.addStretch()
self.vBox_1.addLayout(self.hBox_details)
self.vBox_1.addStretch()
else:
print("ERROR: confirmDeleteDialog wrong type")
self.btn_confirm = QtWidgets.QPushButton("Yes", self)
self.btn_confirm.clicked.connect(self.confirm)
self.btn_cancel = QtWidgets.QPushButton("No", self)
self.btn_cancel.clicked.connect(self.exit)
self.hBox_1 = QtWidgets.QHBoxLayout()
self.hBox_1.addWidget(self.btn_confirm)
self.hBox_1.addWidget(self.btn_cancel)
self.vBox_1 = QtWidgets.QVBoxLayout(self)
self.vBox_1.addWidget(self.display_text)
self.vBox_1.addLayout(self.hBox_1)
def confirm(self):
if self.type == "Video":
dpr.delete_video(self.id)
elif self.type == "Channel":
dpr.delete_channel(self.id)
self.close()
def exit(self):
self.close()
if __name__ == "__main__":
df_videos = dpr.get_all("Videos")
df_channels = dpr.get_all("Channels")
app = QApplication(sys.argv)
ex = MainFrame()
ex.show()
sys.exit(app.exec_())
#wx is a linear function of the weights and input
wx = w * record.transpose()
#hypothesis, h is true if wx is positive, else it is false
if wx >= 0:
h = 1
else:
h = 0
#if hpyothesis and target labels don't match, then the example has been misclassified
if h != y:
misclassified.append(row)
if len(misclassified) != 0:
#pick random misclassified point
point = choice(misclassified)
record = [data.item(point, i) for i in range(len(attr))]
w = w + np.matrix(record[target_attr]) * np.matrix(record[0:len(attr)])
count += 1
else:
#no misclassifications, Perceptron has attained convergence
converged = 1
if not converged:
print("No convergence")
if __name__ == "__main__":
data = np.matrix(DataProcessing.process("data.txt","processedData.txt"))
perceptron(data, range(14), 13)
def setupDataDisplay(self, indices=range(1,15)):
"""requires that you have already set a number of things on self"""
self.indices = indices
self.displayData = DataProcessing.getDisplayData(self.rawData, self.startTime, self.endTime, self.storedAmplitude, self.lowPass, self.highPass, self.indices)
self.setupZoom(self.displayData)
self.channels = self.rawData.ch_names
self.samplingRate = self.rawData.info['sfreq']
displayChannels = [self.channels[i] for i in indices]
displayPositions = np.linspace(0.9,-0.9,len(displayChannels))
self.annotationTimes = []
self.positionsToTextMap = {}
for i in range(0,len(displayPositions)):
self.positionsToTextMap[(-0.97,displayPositions[i])]=str(displayChannels[i])
self.program = gloo.Program(SERIES_VERT_SHADER, SERIES_FRAG_SHADER)
self.vertices = gloo.VertexBuffer(V)
self.zoomBoxBuffer = gloo.VertexBuffer(V3)
self.progAnnotations = gloo.Program(progAnnotations_vertex_shader, progAnnotations_fragment_shader)
self.progAnnotations.bind(self.vertices)
self.progZoom = gloo.Program(zoombox_vertex_shader, zoombox_fragment_shader)
self.progZoom.bind(self.zoomBoxBuffer)
self.setupZoomStep2()
self.myTextDrawer = TextDrawer(self.physical_size[1],self.physical_size[0])
self.myLineDrawer = LineDrawer()
print "height " + str(self.physical_size[1])
self.progText = gloo.Program(text_vertex_shader, text_fragment_shader)
self.fontBMP= vispy.io.imread("FixedSys.bmp")
tvar = np.array(self.fontBMP[:,:,0])
self.fontBMP=np.dstack((self.fontBMP,tvar))
self.fontTexture =gloo.Texture2D(self.fontBMP,format="rgba")
self.textVerticesArr = self.myTextDrawer.computeTextsData(self.positionsToTextMap)
self.updateLines()
self.textVertices = gloo.VertexBuffer(self.textVerticesArr)
self.progText.bind(self.textVertices)
self.progText["myTextureSampler"] = self.fontTexture
gloo.set_viewport(0, 0, *self.physical_size)
self.updateTextBoxes()
self.dragZoom = False
self.oldPos = None
self.newPos = None
self.orig = None
self.new = None
self.posDiff = None
self.events.mouse_press.connect((self, 'mouse_press'))
self.events.mouse_release.connect((self, 'mouse_release'))
self.events.mouse_move.connect((self, 'on_mouse_move'))
# self._timer = app.Timer('auto', connect=self.on_timer, start=True)
gloo.set_state(clear_color='black', blend=True,
blend_func=('src_alpha', 'one_minus_src_alpha'))
self.update()
def setupChannels(self):
self.tab_channels = QtWidgets.QStackedWidget()
self.tab_channels.setObjectName("tab_channels")
#--- Channel Stats
self.stk_channel_stats = QtWidgets.QWidget()
self.vBox_channel_stats_1 = QtWidgets.QVBoxLayout(self.stk_channel_stats)
self.hBox_channel_stats_1 = QtWidgets.QHBoxLayout()
if dpr.datafile_exists("channel_views.csv") and dpr.datafile_exists("channels.csv"):
self.gra_channel_subs = LineGraphCanvas(
dpr.get_sums_ot(r"channel_subscribers.csv"), "Reached Subscribers Over Time")
self.hBox_channel_stats_1.addWidget(self.gra_channel_subs)
self.gra_channels_count = LineGraphCanvas(
dpr.get_num_items_ot(r"channel_subscribers.csv"), "Number of Channels Over Time")
self.hBox_channel_stats_1.addWidget(self.gra_channels_count)
self.vBox_channel_stats_1.addLayout(self.hBox_channel_stats_1)
self.btn_goto_channel_list = QtWidgets.QPushButton("Channel List", self.stk_channel_stats)
self.btn_goto_channel_list.clicked.connect(self.goto_channel_list)
self.vBox_channel_stats_1.addWidget(self.btn_goto_channel_list)
self.tab_channels.addWidget(self.stk_channel_stats)
#--- Channel List
self.stk_channel_list = QtWidgets.QWidget()
self.hBox_channel_list_1 = QtWidgets.QHBoxLayout(self.stk_channel_list)
self.hBox_channel_list_1.setObjectName("hBox_channel_list_1")
self.vBox_channel_list_1 = QtWidgets.QVBoxLayout()
self.vBox_channel_list_1.setObjectName("vBox_channel_list_1")
self.hBox_channel_list_2 = QtWidgets.QHBoxLayout()
self.hBox_channel_list_2.setObjectName("hBox_channel_list_2")
self.lbl_channel_list_popularity = QtWidgets.QLabel("Sort:")
self.hBox_channel_list_2.addWidget(self.lbl_channel_list_popularity)
self.cmb_channel_list_popularity = QtWidgets.QComboBox(self.stk_channel_list)
self.cmb_channel_list_popularity.setObjectName("cmb_channel_list_popularity")
self.channel_popularity = ["Subscribers", "Most Recent", "Channel Name"]
for i in self.channel_popularity:
self.cmb_channel_list_popularity.addItem(i)
self.hBox_channel_list_2.addWidget(self.cmb_channel_list_popularity)
self.cmb_channel_list_popularity.currentIndexChanged.connect(self.cmb_channel_popularity)
spacerItem = QtWidgets.QSpacerItem(
40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
self.hBox_channel_list_2.addItem(spacerItem)
self.vBox_channel_list_1.addLayout(self.hBox_channel_list_2)
self.tbl_channel_list = TableFromDf(df_channels)
self.tbl_channel_list.setObjectName("tlb_channel_list")
self.tbl_channel_list.itemDoubleClicked.connect(self.tbl_show_channel)
self.vBox_channel_list_1.addWidget(self.tbl_channel_list)
self.btn_goto_channel_stats = QtWidgets.QPushButton(
"Channel Stats", self.stk_channel_list)
self.btn_goto_channel_stats.clicked.connect(self.goto_channel_stats)
self.vBox_channel_list_1.addWidget(self.btn_goto_channel_stats)
self.hBox_channel_list_1.addLayout(self.vBox_channel_list_1)
self.tab_channels.addWidget(self.stk_channel_list)
def cmb_video_region(self, index):
dpr.filter_video_region(self.cmb_video_list_region.itemText(index))
def cmb_video_popularity(self, index):
self.tbl_video_list.populate(dpr.prep_table_dsp(dpr.sort_videos(self.cmb_video_list_popularity.itemText(index), self.tbl_video_list.df)))
self.tbl_video_list.df = dpr.sort_videos(self.cmb_video_list_popularity.itemText(index), self.tbl_video_list.df)
def cmb_channel_region(self, index):
dpr.filter_channel_region(self.cmb_channel_list_region.itemText(index))
import DataProcessing as dp
import network as NN
train = dp.get_data_train("mnist_train.csv")
test = dp.get_data_test("mnist_test.csv")
print "*************************"
print "* train and test loaded *"
print "*************************"
nn = NN.Network([784,30,10])
print "*************************"
print "* nn loaded *"
print "*************************"
nn.SGD(train,30,10,0.1,test_data=test)
def __init__(self, video):
QHBoxLayout.__init__(self)
self.id = video.name
self.vid_thumbdata = dpr.get_thumb_data(self.id, "Video")
self.img_vid = QtGui.QPixmap()
self.img_vid.loadFromData(self.vid_thumbdata)
self.img_vid = self.img_vid.scaledToHeight(80)
self.lbl_vid = QtWidgets.QLabel()
self.lbl_vid.mouseReleaseEvent = self.goToVideo
self.lbl_vid.setPixmap(self.img_vid)
self.addWidget(self.lbl_vid)
self.vBox_all = QtWidgets.QVBoxLayout()
self.vBox_all.addStretch()
self.lbl_name = QtWidgets.QLabel(video["Title"], font=f_subheader)
self.vBox_all.addWidget(self.lbl_name)
self.hBox_top = QtWidgets.QHBoxLayout()
self.vBox_views = QtWidgets.QVBoxLayout()
self.lbl_views = QtWidgets.QLabel("Views")
self.vBox_views.addWidget(self.lbl_views)
self.lbl_views_val = QtWidgets.QLabel(str(video["Views"]))
self.vBox_views.addWidget(self.lbl_views_val)
self.hBox_top.addLayout(self.vBox_views)
self.vBox_channel = QtWidgets.QVBoxLayout()
self.lbl_channel = QtWidgets.QLabel("Channel")
self.vBox_channel.addWidget(self.lbl_channel)
self.lbl_channel_val = QtWidgets.QLabel(video["ChannelTitle"])
self.vBox_channel.addWidget(self.lbl_channel_val)
self.hBox_top.addLayout(self.vBox_channel)
self.vBox_likes = QtWidgets.QVBoxLayout()
self.lbl_likes = QtWidgets.QLabel("Likes/Dislikes")
self.vBox_likes.addWidget(self.lbl_likes)
self.lbl_likes_val = QtWidgets.QLabel(str(video["Likes"])+"/"+str(video["Dislikes"]))
self.vBox_likes.addWidget(self.lbl_likes_val)
self.hBox_top.addLayout(self.vBox_likes)
self.vBox_duration = QtWidgets.QVBoxLayout()
self.lbl_duration = QtWidgets.QLabel("Duration")
self.vBox_duration.addWidget(self.lbl_duration)
self.lbl_duration_val = QtWidgets.QLabel(str(dpr.parse_duration(video["Duration"])))
self.vBox_duration.addWidget(self.lbl_duration_val)
self.hBox_top.addLayout(self.vBox_duration)
self.vBox_comments = QtWidgets.QVBoxLayout()
self.lbl_comments = QtWidgets.QLabel("Comments")
self.vBox_comments.addWidget(self.lbl_comments)
self.lbl_comments_val = QtWidgets.QLabel(str(video["Comments"]))
self.vBox_comments.addWidget(self.lbl_comments_val)
self.hBox_top.addLayout(self.vBox_comments)
self.vBox_upload = QtWidgets.QVBoxLayout()
self.lbl_upload = QtWidgets.QLabel("Upload")
self.vBox_upload.addWidget(self.lbl_upload)
self.lbl_upload_val = QtWidgets.QLabel(dpr.parse_date(video["Upload"]))
self.vBox_upload.addWidget(self.lbl_upload_val)
self.hBox_top.addLayout(self.vBox_upload)
self.vBox_gathered = QtWidgets.QVBoxLayout()
self.lbl_gathered = QtWidgets.QLabel("Gathered")
self.vBox_gathered.addWidget(self.lbl_gathered)
self.lbl_gathered_val = QtWidgets.QLabel(dpr.parse_date(video["Gathered"]))
self.vBox_gathered.addWidget(self.lbl_gathered_val)
self.hBox_top.addLayout(self.vBox_gathered)
self.vBox_all.addLayout(self.hBox_top)
self.vBox_all.addStretch()
self.addLayout(self.vBox_all)
self.addStretch()
def cmb_channel_timeframe(self, index):
dpr.filter_channel_timeframe(self.cmb_channel_list_timeframe.itemText(index))
def __init__(self, channel):
QHBoxLayout.__init__(self)
self.id = channel.name
self.chan_thumbdata = dpr.get_thumb_data(self.id, "Channel")
self.img_chan = QtGui.QPixmap()
self.img_chan.loadFromData(self.chan_thumbdata)
self.img_chan = self.img_chan.scaledToHeight(80)
self.lbl_chan = QtWidgets.QLabel()
self.lbl_chan.mouseReleaseEvent = self.goToChannel
self.lbl_chan.setPixmap(self.img_chan)
self.addWidget(self.lbl_chan)
self.vBox_all = QtWidgets.QVBoxLayout()
self.vBox_all.addStretch()
self.lbl_name = QtWidgets.QLabel(channel["Title"], font=f_subheader)
self.vBox_all.addWidget(self.lbl_name)
self.hBox_top = QtWidgets.QHBoxLayout()
self.vBox_subscribers = QtWidgets.QVBoxLayout()
self.lbl_subscribers = QtWidgets.QLabel("Subscribers")
self.vBox_subscribers.addWidget(self.lbl_subscribers)
self.lbl_subscribers_val = QtWidgets.QLabel(str(channel["Subscribers"]))
self.vBox_subscribers.addWidget(self.lbl_subscribers_val)
self.hBox_top.addLayout(self.vBox_subscribers)
self.vBox_views = QtWidgets.QVBoxLayout()
self.lbl_views = QtWidgets.QLabel("Views")
self.vBox_views.addWidget(self.lbl_views)
self.lbl_views_val = QtWidgets.QLabel(str(channel["Views"]))
self.vBox_views.addWidget(self.lbl_views_val)
self.hBox_top.addLayout(self.vBox_views)
self.vBox_num_vids = QtWidgets.QVBoxLayout()
self.lbl_num_vids = QtWidgets.QLabel("# Videos")
self.vBox_num_vids.addWidget(self.lbl_num_vids)
self.lbl_num_vids_val = QtWidgets.QLabel(str(len(dpr.get_scanned_videos(self.id))))
self.vBox_num_vids.addWidget(self.lbl_num_vids_val)
self.hBox_top.addLayout(self.vBox_num_vids)
self.vBox_comments = QtWidgets.QVBoxLayout()
self.lbl_comments = QtWidgets.QLabel("Comments")
self.vBox_comments.addWidget(self.lbl_comments)
self.lbl_comments_val = QtWidgets.QLabel(str(channel["Comments"]))
self.vBox_comments.addWidget(self.lbl_comments_val)
self.hBox_top.addLayout(self.vBox_comments)
self.vBox_upload = QtWidgets.QVBoxLayout()
self.lbl_upload = QtWidgets.QLabel("Upload")
self.vBox_upload.addWidget(self.lbl_upload)
self.lbl_upload_val = QtWidgets.QLabel(dpr.parse_date(channel["Upload"]))
self.vBox_upload.addWidget(self.lbl_upload_val)
self.hBox_top.addLayout(self.vBox_upload)
self.vBox_gathered = QtWidgets.QVBoxLayout()
self.lbl_gathered = QtWidgets.QLabel("Gathered")
self.vBox_gathered.addWidget(self.lbl_gathered)
self.lbl_gathered_val = QtWidgets.QLabel(dpr.parse_date(channel["Gathered"]))
self.vBox_gathered.addWidget(self.lbl_gathered_val)
self.hBox_top.addLayout(self.vBox_gathered)
self.vBox_all.addLayout(self.hBox_top)
self.vBox_all.addStretch()
self.addLayout(self.vBox_all)
self.addStretch()
def show_tfr_plot(self):
DataProcessing.generate_and_plot_waveletAnalysis(self.canvas.rawData,self.canvas.indices[0],self.canvas.startTime,self.canvas.endTime,self.canvas.samplingRate)
def setupAddAndScan(self):
self.tab_add_scan = QtWidgets.QWidget()
self.tab_add_scan.setEnabled(True)
self.tab_add_scan.setAccessibleName("")
self.tab_add_scan.setObjectName("tab_add_scan")
self.hBox_add_scan_2 = QtWidgets.QHBoxLayout(self.tab_add_scan)
self.hBox_add_scan_2.setObjectName("hBox_add_scan_2")
self.vBox_add_scan_1 = QtWidgets.QVBoxLayout()
self.vBox_add_scan_1.setObjectName("vBox_add_scan_1")
self.hBox_add_scan_1 = QtWidgets.QHBoxLayout()
self.hBox_add_scan_1.setObjectName("hBox_add_scan_1")
self.line_add_url = QtWidgets.QLineEdit(self.tab_add_scan)
self.line_add_url.setFixedSize(350,22)
self.line_add_url.setObjectName("line_add_url")
self.line_add_url.setText("Insert URL")
self.hBox_add_scan_1.addWidget(self.line_add_url)
self.btn_add_video = QtWidgets.QPushButton(self.tab_add_scan)
self.btn_add_video.setToolTipDuration(1)
self.btn_add_video.setObjectName("btn_add_video")
self.btn_add_video.setText("Add Video")
self.btn_add_video.setToolTip("Add a video to the database.")
self.btn_add_video.clicked.connect(self.btn_add_video_press)
self.hBox_add_scan_1.addWidget(self.btn_add_video)
self.btn_add_channel = QtWidgets.QPushButton(self.tab_add_scan)
self.btn_add_channel.setToolTipDuration(1)
self.btn_add_channel.setObjectName("btn_add_channel")
self.btn_add_channel.setText("Add Channel")
self.btn_add_channel.setToolTip("Add a chanel to the database")
self.btn_add_channel.clicked.connect(self.btn_add_channel_press)
self.hBox_add_scan_1.addWidget(self.btn_add_channel)
self.hBox_add_scan_1.addStretch()
self.vBox_add_scan_1.addLayout(self.hBox_add_scan_1)
self.grid = QtWidgets.QGridLayout()
self.vBox_recent_videos = QtWidgets.QVBoxLayout()
self.lbl_recent_videos = QtWidgets.QLabel("Last Uploaded Videos", font=f_header)
self.vBox_recent_videos.addWidget(self.lbl_recent_videos)
self.recent_videos = dpr.get_sorted(df_videos, "Upload", 3)
self.recent_video_details = [videoDetails(self.recent_videos.iloc[x]) for x in range(len(self.recent_videos))]
for i in range(len(self.recent_video_details)):
self.vBox_recent_videos.addLayout(self.recent_video_details[i])
self.grid.addLayout(self.vBox_recent_videos, 0,0)
self.vBox_add_scan_video_details = QtWidgets.QVBoxLayout()
self.vBox_add_scan_video_details.addStretch()
self.lbl_all_videos = QtWidgets.QLabel("Total Videos: " + str(dpr.get_num_items(df_videos)), font=f_preview)
self.vBox_add_scan_video_details.addWidget(self.lbl_all_videos)
self.lbl_all_video_views = QtWidgets.QLabel("Total Video Views: " + str(dpr.get_latest_sum(r"video_views.csv")), font=f_preview)
self.vBox_add_scan_video_details.addWidget(self.lbl_all_video_views)
self.lbl_all_likes_dislikes = QtWidgets.QLabel("Total Likes/Dislikes: " + str(dpr.get_latest_sum(r"video_likes.csv")) + "/" + str(dpr.get_latest_sum(r"video_dislikes.csv")), font=f_preview)
self.vBox_add_scan_video_details.addWidget(self.lbl_all_likes_dislikes)
self.lbl_dur_total = QtWidgets.QLabel("Runtime of all Videos combined: " + str(dpr.get_dur_total()), font=f_preview)
self.vBox_add_scan_video_details.addStretch()
self.hBox_stretch_videos = QtWidgets.QHBoxLayout()
self.hBox_stretch_videos.addLayout(self.vBox_add_scan_video_details)
self.hBox_stretch_videos.addStretch()
self.grid.addLayout(self.hBox_stretch_videos, 0,1)
self.vBox_recent_channels = QtWidgets.QVBoxLayout()
self.lbl_recent_channels = QtWidgets.QLabel("Last Added Channels", font=f_header)
self.vBox_recent_channels.addWidget(self.lbl_recent_channels)
self.recent_channels = dpr.get_sorted(df_channels, "Gathered", 3)
self.recent_channel_details = [channelDetails(self.recent_channels.iloc[x]) for x in range(len(self.recent_channels))]
for i in range(len(self.recent_channel_details)):
self.vBox_recent_channels.addLayout(self.recent_channel_details[i])
self.grid.addLayout(self.vBox_recent_channels, 1,0)
self.vBox_add_scan_channel_details = QtWidgets.QVBoxLayout()
self.vBox_add_scan_channel_details.addStretch()
self.lbl_all_channels = QtWidgets.QLabel("Total Channels: " + str(dpr.get_num_items(df_channels)), font=f_preview)
self.vBox_add_scan_channel_details.addWidget(self.lbl_all_channels)
self.lbl_all_subscribers = QtWidgets.QLabel("Total Subscribers: " + str(dpr.get_latest_sum(r"channel_subscribers.csv")), font=f_preview)
self.vBox_add_scan_channel_details.addWidget(self.lbl_all_subscribers)
self.vBox_add_scan_channel_details.addStretch()
self.grid.addLayout(self.vBox_add_scan_channel_details, 1,1)
self.vBox_add_scan_1.addLayout(self.grid)
spacerItem = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
self.vBox_add_scan_1.addItem(spacerItem)
self.hBox_add_scan_2.addLayout(self.vBox_add_scan_1)
#try to match the IDs of all videos or channels with the IDs of the data from gather_data.
self.stats = {}
for i in self.id_list:
try:
self.stats[i] = self.data[i][stat]
except KeyError as e:
self.stats[i] = "NaN"
self.df_stats = pd.DataFrame.from_dict(self.stats, orient="index", columns=[self.curr_time])
#if there is no previous channels- or videos-file or if they are empty, don't merge with df_stats before writing
try:
self.df_old = dpr.get_dataframe(output_file)
self.df_new = self.df_old.join(self.df_stats, how="outer")
dpr.write_dataframetofile(self.df_new, output_file)
except (ValueError, OSError):
dpr.write_dataframetofile(self.df_stats, output_file)
if __name__ == "__main__":
channels = DataContainer("channels", "channels.csv")
dpr.update_videos(channels.id_list) #Look for new Videos in all channels before updating the rest
channels.gather_data()
channels.update_stats("channel_views.csv", "viewCount")
channels.update_stats("channel_subscribers.csv", "subscriberCount")
channels.update_stats("channel_videos.csv", "videoCount")
channels.update_stats("channel_comments.csv", "commentCount")
videos = DataContainer("videos", "videos.csv")
videos.gather_data()
videos.update_stats("video_views.csv", "viewCount")
videos.update_stats("video_likes.csv", "likeCount")
videos.update_stats("video_dislikes.csv", "dislikeCount")
videos.update_stats("video_comments.csv", "commentCount")
def cmb_video_timeframe(self, index):
dpr.filter_video_timeframe(self.cmb_video_list_timeframe.itemText(index))
def cmb_channel_popularity(self, index):
self.tbl_channel_list.populate(dpr.prep_table_dsp(dpr.sort_channels(self.cmb_channel_list_popularity.itemText(index), self.tbl_channel_list.df)))
self.tbl_channel_list.df = dpr.sort_channels(self.cmb_channel_list_popularity.itemText(index), self.tbl_channel_list.df)
def confirm(self):
if self.type == "Video":
dpr.delete_video(self.id)
elif self.type == "Channel":
dpr.delete_channel(self.id)
self.close()
