def on_actRun_triggered(self):
"""
运行按钮
:return: 无返回值
"""
self.ui.actRun.setEnabled(False) # 这里需要注意的是点击一次run 控件之后，应当设置未为不可选，
self.key_para["SaveData_Statue"] = False
self.key_para.update(self.get_panel_para()) # 运行前应该得到面板参数，返回dict，并更新self.key_para
print(self.key_para)
self._data_thread = QThread()
self.data_analysis = DataAnalysis(self.key_para)
self.data_analysis.pbar.connect(self.set_progressBar_int)
self.data_analysis.tbw.connect(self.add_textBrowser_str)
self.data_analysis.sbar.connect(self.add_statusBar_str)
self.data_analysis.run_end.connect(lambda: self.stop_thread(self._data_thread))
self.data_analysis.moveToThread(self._data_thread)
self._data_thread.started.connect(self.data_analysis.run)
self._data_thread.finished.connect(self.draw_prepare)
show_info = "数据计算中..."
self.add_statusBar_str(show_info)
self.add_textBrowser_str(show_info)
self._data_thread.start()
print(f'开启{self._data_thread.currentThread()}线程，现在线程状态 ：', self._data_thread.isRunning())
def analyze_and_train():
# to view the different metrics obtained from the data run perform data analysis
eda = DataAnalysis(input)
eda.perform_data_analysis()
# to train a model run the trainer function
tr = Trainer(data_path=input, model_name=model_name)
tr.train()
tr.eval()
tr.save()
def main(file_name):
data = DataAnalysis(file_name)
# Report top ten languages by frequency
print("Languages:")
print_output(data.top_n_lang_freqs(10))
# Report top ten country (2 letter) top
# level domains by frequency
print("Top level country domains:")
print_output(data.top_n_country_tlds_freqs(10))
def main(file_name):
data = DataAnalysis(file_name)
data.read_data(file_name)
print("Languages:")
print_output(data.top_n_lang_freqs(10))
print("Top level country domains:")
print_output(data.top_n_country_tlds_freqs(10))
def _main_thread_loop():
global thread_DH
global thread_DA1
global thread_DBStrip
global thread_J
global packet_queue
# Поднимаем потоки
thread_DH.start()
thread_DA1.start()
thread_DBStrip.start()
thread_J.start()
# Перезапускаем потоки по мере их падения
while True:
if thread_DH.isAlive() == False:
thread_DH = DataHarvest(packet_queue)
thread_DH.start()
logger.warning('Поток сборщика данных перезапущен.')
if thread_DA1.isAlive() == False:
thread_DA1 = DataAnalysis(packet_queue)
thread_DA1.start()
logger.warning('Поток анализа данных канала перезапущен.')
if thread_DBStrip.isAlive() == False:
thread_DBStrip = DBStriper()
thread_DBStrip.start()
logger.warning('Поток удаления старых данных перезапущен.')
if thread_J.isAlive() == False:
thread_J = EventJournal()
thread_J.start()
logger.warning('Поток журналирования перезапущен.')
sleep(THREAD_CHECK_TIMEOUT)
def main(file_name):
'''analyze the users file for top usage of languages
and top level domain'''
data = DataAnalysis(file_name)
data.read_data(file_name)
# Report top ten languages by frequency
print("Languages:")
print_output(data.top_n_lang_freqs(10))
# Report top ten country (2 letter) top
# level domains by frequency
print("Top level country domains:")
print_output(data.top_n_country_tlds_freqs(10))
def splitDataFirstMode():
# use all available data
sampleData = dp.returnData(1, randomSeed=10)
saveDataset(sampleData, 'AD_dataset')
# split data into X and y
da = DataAnalysis(dirName=dirName)
X, y = da.splitXY(sampleData)
# split data into training (80%) and testing (20%) set
xTrain, xTest, yTrain, yTest = da.splitTrainTest(X, y, trainSize=0.8)
saveDataset(xTrain.assign(normality=yTrain.values), 'AD_set_train')
saveDataset(xTest.assign(normality=yTest.values), 'AD_set_test')
# get data characteristics for current dataset size
dataInfo[1] = da.getDataCharacteristics(yTrain, yTest)
# Run predictions
for datasetSize in (selectedSizes or [0.2, 0.4, 0.6, 0.8, 1]):
# get no. of samples and prediction accuracy
noOfSamples, predictions = da.getScores(
xTrain,
xTest,
yTrain,
yTest,
trainSize=datasetSize,
randomSeeds=randomSeeds,
selectedAlgorithms=selectedAlgorithms
) if selectedAlgorithms else da.getScores(xTrain,
yTrain,
xTest,
yTest,
trainSize=datasetSize,
randomSeeds=randomSeeds)
# save results for graphs and .csv files
savePredictionScores(noOfSamples, predictions, datasetSize)
def normalMode():
da = DataAnalysis(dirName=dirName)
# Run predictions
for datasetSize in (selectedSizes or [0.2, 0.4, 0.6, 0.8, 1]):
# get the percentage of all data
sampleData = dp.returnData(datasetSize, randomSeed=10)
saveDataset(sampleData, 'AD_dataset')
# split data into X and y
X, y = da.splitXY(sampleData)
# split data into training (80%) and testing (20%) set
xTrain, xTest, yTrain, yTest = da.splitTrainTest(X, y, trainSize=0.8)
# get data characteristics for current dataset size
dataInfo[datasetSize] = da.getDataCharacteristics(yTrain, yTest)
# get no. of samples and prediction accuracy (trainSize is set to 1, so no further data splitting is done)
# multiple runs make no sense here, since we always take whole set
noOfSamples, predictions = da.getScores(
xTrain,
xTest,
yTrain,
yTest,
trainSize=1,
randomSeeds=[10],
selectedAlgorithms=selectedAlgorithms
) if selectedAlgorithms else da.getScores(xTrain,
yTrain,
xTest,
yTest,
trainSize=datasetSize,
randomSeeds=randomSeeds)
# save results for graphs and .csv files
savePredictionScores(noOfSamples, predictions, datasetSize)
def __load_file(self):
"""
loading selected file and will launch the preliminary analysis
Return:
None
"""
if self.fileName and not hasattr(self, 'da'):
self.da = DataAnalysis.create_object(file_path=self.fileName)
self.da.get_statistics()
stats = self.da.stats
self.grid_widget = QWidget()
grid = QGridLayout()
grid.setObjectName('grid_stats')
self.fields = stats.columns
self.qties = stats.index
self.active_buttons = dict()
for indf, field in enumerate(self.fields, 1):
self.active_buttons[field] = False
btn = LabelButton.create_label_button(label=field,
window=self,
connect_fns=[])
grid.addWidget(btn, indf, 0)
for indq, qty in enumerate(self.qties, 1):
grid.addWidget(QLabel(qty), 0, indq)
for indf, field in enumerate(self.fields):
for indq, qty in enumerate(self.qties):
grid.addWidget(QLabel(str(stats.iloc[indq, indf])),
1 + indf, 1 + indq)
grid.setColumnStretch(indq + 1, 1)
self.grid_widget.setLayout(grid)
self.main_layout.addWidget(self.grid_widget)
# self.main_layout.addLayout(grid, 1, 0, 1, 2)
if not hasattr(self.hbox, 'btn_draw'):
self.btn_draw = UserInterface._create_button(
text='Draw Chart',
icon_path='icons/draw2.png',
connect_fn=self.__draw_chart)
self.hbox.addWidget(self.btn_draw)
else:
self.btn_draw.setVisible(True)
if not hasattr(self.hbox, 'btn_unload'):
self.btn_unload = UserInterface._create_button(
text='Unload data',
icon_path='icons/remove2.png',
connect_fn=self.__remove)
self.hbox.addWidget(self.btn_unload)
else:
self.btn_unload.setVisible(True)
self.hbox.addStretch()
self.hbox.setDirection(QBoxLayout.LeftToRight)
self.statusBar().showMessage(self.os_text +
'\t\t\tData file loaded.')
else:
self.statusBar().showMessage(self.os_text +
'\t\t\tNo file selected')
self.update()
# Load config
config = json.load(open("../config.json"))
# Load markets/articles
articles_csv = csv.reader(open("articles.csv"))
# Parse markets/articles
articles = []
for row in articles_csv:
if row[0] == "Article Title":
# Skip header
continue
article_obj = {}
article_obj["article"] = {"title": row[0]}
article_obj["market"] = {
"symbol": "TEST_MARKET",
"entities": row[1].split(","),
"wikipedia_urls": row[2].split(","),
"target_words": row[3].split(","),
"anti_target_words": row[4].split(",")
}
article_obj["label"] = int(row[5])
articles.append(article_obj)
da = DataAnalysis(config, load_model=False)
da.create_model(articles)
da.load_model()
import sys import matplotlib.pyplot as plt import intrinio from data_analysis import DataAnalysis # USERNAME = '******' # PASSWORD = '******' USERNAME = '******' PASSWORD = '******' data_collection = DataCollection(username=USERNAME, password=PASSWORD) ticker_symbol = "AAPL" dates, prices = data_collection.retrieve_data(ticker_symbol) da = DataAnalysis() # data = da.get_yearly_trends(dates, prices, [3]) bounds, sell_dates, profits = da.get_optimal_bounds(dates, prices, low_bound_min=0.3, top_bound_max=0.5, interval=0.01, investment=100, purchase_strategy="immediate") da.plot_data(dates, prices, ticker_symbol, sell_dates=sell_dates) # if sys.argv[1]: # ticker = sys.argv[1] # start_date = sys.argv[2] # results = data_collection.get_prices(ticker, start_date) # data_collection.plot_data(ticker) # data_collection.save_data(ticker)
import threading
import logging
from logging.handlers import RotatingFileHandler
from time import sleep
from sqlalchemy import create_engine
from sqlalchemy_utils import database_exists, create_database
import os
from param import extractParams
appdir = os.path.abspath(os.path.dirname(__file__))
logger = logging.getLogger("mitter")
packet_queue = Queue()
thread_DH = DataHarvest(packet_queue)
thread_DA1 = DataAnalysis(packet_queue)
thread_DBStrip = DBStriper()
thread_J = EventJournal()
THREAD_CHECK_TIMEOUT = 1 # Интервал контроля работоспособности потоков в сек.
def MainRoutine():
_logger_config()
_postgres_db_check()
extractParams()
try:
_main_thread_loop()
except Exception:
class QmyBasicAnalysisModule(QMainWindow):
def __init__(self, parent=None):
super(QmyBasicAnalysisModule, self).__init__(parent)
self.ui = Ui_QWBasicAnalysisModule()
self.ui.setupUi(self)
self.key_para = {} # 页面关键参数dict
self.init_set() # 初始化设置
def init_set(self):
self.DRAW_FIRST = True
self.key_para["SaveData_Statue"] = False # 此参数标志是否可以进行数据保存的工作，应当在得到绘图数据之后设置为True，并且在每点击一次run之后设置为False
self.key_para["Data_Save_Path"] = ""
self.init_widget()
def init_widget(self):
self.add_textBrowser_str("*" * 26 + "欢迎使用" + "*" * 26, showtime=False)
self.add_statusBar_str("请先加载数据文件......")
self._2D_condutance_layout = QVBoxLayout(self)
self._1D_condutance_layout = QVBoxLayout(self)
self._1D_length_layout = QVBoxLayout(self)
self.init_save_dir()
self.check_config()
self.ui.actRun.setEnabled(False)
self.ui.actStop.setEnabled(False)
self.ui.btn_Redraw.setEnabled(False) # 重画按钮，应当在绘图成功后设置为可触发
self.ui.btn_Update.setEnabled(False) # 更新additional-length按钮，应当在绘图成功后设置为可触发
self.ui.btn_SaveResult.setEnabled(
self.check_savedir_statue()) # 保存图片及对应作图文件按钮，初次使用应该是false，后面继续使用，应该会用config中自动设置上次的路径，所以应该加一个判断函数
# =============== 控件触发函数===============
@pyqtSlot()
def on_actOpenFiles_triggered(self):
"""
选择文件act
:return:
"""
dlg_title = "选择多个文件" # 对话框标题
filt = "TDMS Files(*.tdms)" # 文件过滤器
desptop_path = get_desktop_path()
load_statue = False
while not load_statue:
file_list, filt_used = QFileDialog.getOpenFileNames(self, dlg_title, desptop_path, filt)
load_statue = self.set_load_state(file_list)
if not load_statue:
result = QMessageBox.warning(self, "警告", "请至少选择一个文件!", QMessageBox.Ok | QMessageBox.Cancel,
QMessageBox.Ok)
if result == QMessageBox.Cancel:
break
self.add_textBrowser_str("加载失败，请至少选择一个文件！点击打开重新选择文件。")
self.add_statusBar_str("加载失败，请至少选择一个文件！点击打开重新选择文件。")
else:
self.key_para["LOAD_STATUE"] = load_statue
self.key_para['FILE_PATHS'] = file_list
self.add_textBrowser_str(f"选择文件(数量:{len(file_list)})：")
self.add_textBrowser_list(file_list)
self.add_textBrowser_str("*" * 60, showtime=False)
# 加载文件成功之后，应当对运行按钮进行释放
self.ui.actRun.setEnabled(True)
@pyqtSlot()
def on_actQuit_triggered(self):
"""
这个就离谱，我只是调用了close函数，为什么又调用了我重写的closeevent？？？？
:return:
"""
# todo 需要想明白！！！这里虽然功能没问题，但是这是为什么呢？？？
self.close()
def closeEvent(self, event):
"""
重写窗口关闭函数，关闭前保存面板参数
:param event: 无
:return: 无
"""
dlg_title = "警告"
str_info = "确定退出吗?"
reply = QMessageBox.question(self, dlg_title, str_info,
QMessageBox.Yes | QMessageBox.Cancel,
QMessageBox.Cancel)
if reply == QMessageBox.Yes:
self.save_config_para()
self.add_statusBar_str("参数保存中......")
time.sleep(0.1)
event.accept()
else:
event.ignore()
@pyqtSlot()
def on_actRun_triggered(self):
"""
运行按钮
:return: 无返回值
"""
self.ui.actRun.setEnabled(False) # 这里需要注意的是点击一次run 控件之后，应当设置未为不可选，
self.key_para["SaveData_Statue"] = False
self.key_para.update(self.get_panel_para()) # 运行前应该得到面板参数，返回dict，并更新self.key_para
print(self.key_para)
self._data_thread = QThread()
self.data_analysis = DataAnalysis(self.key_para)
self.data_analysis.pbar.connect(self.set_progressBar_int)
self.data_analysis.tbw.connect(self.add_textBrowser_str)
self.data_analysis.sbar.connect(self.add_statusBar_str)
self.data_analysis.run_end.connect(lambda: self.stop_thread(self._data_thread))
self.data_analysis.moveToThread(self._data_thread)
self._data_thread.started.connect(self.data_analysis.run)
self._data_thread.finished.connect(self.draw_prepare)
show_info = "数据计算中..."
self.add_statusBar_str(show_info)
self.add_textBrowser_str(show_info)
self._data_thread.start()
print(f'开启{self._data_thread.currentThread()}线程，现在线程状态 ：', self._data_thread.isRunning())
@pyqtSlot()
def on_actGuideSet_triggered(self):
# TODO
# 参数设置指南QAction，这里应当弹出一个Dialog给出提示，此处不重要，最后再添加
pass
@pyqtSlot()
def on_actStop_triggered(self):
# TODO
# 终止整个程序运行，清空内存，这里如何编写需要思考
pass
@pyqtSlot(int)
def on_cmb_Device_currentIndexChanged(self, index):
"""
仪器设备combox，此处响应参数为int，我想int应该比string快一点哈哈
:param index: 对应的combox改变时的选项序号
:return: 无返回值
"""
self.key_para["DEVICE_ID"] = index
"""
设备id号对应的设备名
DEVICE_ID DEVICE_NAME
0 stm41
1 stm40
2 mcbj41
3 stm_new
"""
@pyqtSlot(int)
def on_cmb_Process_currentIndexChanged(self, index):
"""
分析过程combox，对应的过程发生改变
:param index: 对应的combox改变时的选项序号
:return: 无
"""
self.key_para["PROCESS"] = index
"""
0 open
1 close
"""
@pyqtSlot(bool)
def on_rdo_select_open_toggled(self, checked):
"""
筛选功能是否打开，默认没有
:param checked: bool值
:return: no
"""
self.key_para["SELECT_OPTION"] = checked
@pyqtSlot()
def on_btn_Update_clicked(self):
self.key_para["SaveData_Statue"] = False
self.key_para["le_Additional_Length"] = int(self.ui.le_Additional_Length.text()) # 更新additional length
ADDITIONAL_LENGTH = self.key_para["le_Additional_Length"]
# 这里需要注意的是，dataset是一个list，因为使用了多进程！！！
for data in self.dataset:
data[3] = data[1] + ADDITIONAL_LENGTH
self.cal_draw()
@pyqtSlot()
def on_btn_Redraw_clicked(self):
self.key_para["SaveData_Statue"] = False
# 此处应该只更新绘图部分的关键参数就可以了吧
key_para = {}
obj_draw_para_list = self.get_same_widget(self.ui.toolBox, QLineEdit)
for obj in obj_draw_para_list:
key_para[obj.objectName()] = float(obj.text())
self.key_para.update(key_para)
self.draw(self.distance_draw, self.condutance_draw, self.length)
@pyqtSlot()
def on_btn_Reset_clicked(self):
_translate = QtCore.QCoreApplication.translate
self.ui.le_Sampling_Rate.setText(_translate("QWBasicAnalysisModule", "20000"))
self.ui.le_Piezo_Rate.setText(_translate("QWBasicAnalysisModule", "1"))
self.ui.le_Stretching_Rate.setText(_translate("QWBasicAnalysisModule", "10"))
self.ui.le_BiasV.setText(_translate("QWBasicAnalysisModule", "0.1"))
self.ui.le_High_Cut.setText(_translate("QWBasicAnalysisModule", "1.2"))
self.ui.le_Low_Cut.setText(_translate("QWBasicAnalysisModule", "-6.2"))
self.ui.le_High_Length.setText(_translate("QWBasicAnalysisModule", "-0.3"))
self.ui.le_Low_Length.setText(_translate("QWBasicAnalysisModule", "-6"))
self.ui.le_Zero_Set.setText(_translate("QWBasicAnalysisModule", "-0.3"))
self.ui.le_Jump_Gap.setText(_translate("QWBasicAnalysisModule", "10000"))
#===============================================================================================
self.ui.le_2D_BinsX.setText(_translate("QWBasicAnalysisModule", "500"))
self.ui.le_2D_BinsY.setText(_translate("QWBasicAnalysisModule", "800"))
self.ui.le_2D_Xleft.setText(_translate("QWBasicAnalysisModule", "-0.2"))
self.ui.le_2D_Xright.setText(_translate("QWBasicAnalysisModule", "1.5"))
self.ui.le_2D_Yleft.setText(_translate("QWBasicAnalysisModule", "-7"))
self.ui.le_2D_Yright.setText(_translate("QWBasicAnalysisModule", "1.5"))
self.ui.le_1D_Cond_Xleft.setText(_translate("QWBasicAnalysisModule", "-7"))
self.ui.le_1D_Cond_Xright.setText(_translate("QWBasicAnalysisModule", "1.5"))
self.ui.le_1D_Cond_Bins.setText(_translate("QWBasicAnalysisModule", "800"))
self.ui.le_1D_Leng_Xleft.setText(_translate("QWBasicAnalysisModule", "0"))
self.ui.le_1D_Leng_Xright.setText(_translate("QWBasicAnalysisModule", "1.5"))
self.ui.le_1D_Leng_Bins.setText(_translate("QWBasicAnalysisModule", "100"))
# ===============================================================================================
self.ui.le_Additional_Length.setText(_translate("QWBasicAnalysisModule", "3000"))
desktop_path = get_desktop_path()
self.ui.le_Data_Save_Dir.setText(_translate("QWBasicAnalysisModule", desktop_path))
# ===============================================================================================
self.ui.le_Start1.setText(_translate("QWBasicAnalysisModule", "-2"))
self.ui.le_Start2.setText(_translate("QWBasicAnalysisModule", "-4"))
self.ui.le_End1.setText(_translate("QWBasicAnalysisModule", "-3"))
self.ui.le_End2.setText(_translate("QWBasicAnalysisModule", "-6"))
self.ui.le_Low_Limit1.setText(_translate("QWBasicAnalysisModule", "-55"))
self.ui.le_Low_Limit2.setText(_translate("QWBasicAnalysisModule", "-55"))
self.ui.le_Upper_Limit1.setText(_translate("QWBasicAnalysisModule", "55"))
self.ui.le_Upper_Limit2.setText(_translate("QWBasicAnalysisModule", "55"))
# ===============================================================================================
self.ui.le_STM41_a1.setText(_translate("QWBasicAnalysisModule", "-9.2694"))
self.ui.le_STM41_b1.setText(_translate("QWBasicAnalysisModule", "-26.0147"))
self.ui.le_STM41_c1.setText(_translate("QWBasicAnalysisModule", "-6.7339e-12"))
self.ui.le_STM41_d1.setText(_translate("QWBasicAnalysisModule", "5.45977e-14"))
self.ui.le_STM41_a2.setText(_translate("QWBasicAnalysisModule", "9.2575"))
self.ui.le_STM41_b2.setText(_translate("QWBasicAnalysisModule", "-25.7897"))
self.ui.le_STM41_c2.setText(_translate("QWBasicAnalysisModule", "3.1833e-12"))
self.ui.le_STM41_d2.setText(_translate("QWBasicAnalysisModule", "6.34969e-15"))
self.ui.le_STM41_offset.setText(_translate("QWBasicAnalysisModule", "0.013"))
# ===============================================================================================
self.ui.le_STM40_a1.setText(_translate("QWBasicAnalysisModule", "-9.1137"))
self.ui.le_STM40_b1.setText(_translate("QWBasicAnalysisModule", "-27.646"))
self.ui.le_STM40_c1.setText(_translate("QWBasicAnalysisModule", "-1.1614e-11"))
self.ui.le_STM40_d1.setText(_translate("QWBasicAnalysisModule", "-1.06185e-13"))
self.ui.le_STM40_a2.setText(_translate("QWBasicAnalysisModule", "9.2183"))
self.ui.le_STM40_b2.setText(_translate("QWBasicAnalysisModule", "-27.8018"))
self.ui.le_STM40_c2.setText(_translate("QWBasicAnalysisModule", "1.1899e-11"))
self.ui.le_STM40_d2.setText(_translate("QWBasicAnalysisModule", "8.05335e-13"))
self.ui.le_STM40_offset.setText(_translate("QWBasicAnalysisModule", "0"))
# ===============================================================================================
self.ui.le_MCBJ41_a1.setText(_translate("QWBasicAnalysisModule", "-9.1316"))
self.ui.le_MCBJ41_b1.setText(_translate("QWBasicAnalysisModule", "-26.9744"))
self.ui.le_MCBJ41_c1.setText(_translate("QWBasicAnalysisModule", "1.8756e-13"))
self.ui.le_MCBJ41_d1.setText(_translate("QWBasicAnalysisModule", "9.8081e-14"))
self.ui.le_MCBJ41_a2.setText(_translate("QWBasicAnalysisModule", "9.1121"))
self.ui.le_MCBJ41_b2.setText(_translate("QWBasicAnalysisModule", "-27.3949"))
self.ui.le_MCBJ41_c2.setText(_translate("QWBasicAnalysisModule", "-7.8635e-13"))
self.ui.le_MCBJ41_d2.setText(_translate("QWBasicAnalysisModule", "2.4551e-13"))
self.ui.le_MCBJ41_offset.setText(_translate("QWBasicAnalysisModule", "-0.025"))
# ===============================================================================================
self.ui.le_STMNEW_a1.setText(_translate("QWBasicAnalysisModule", "-3.9747"))
self.ui.le_STMNEW_b1.setText(_translate("QWBasicAnalysisModule", "-13.265"))
self.ui.le_STMNEW_a2.setText(_translate("QWBasicAnalysisModule", "4.0114"))
self.ui.le_STMNEW_b2.setText(_translate("QWBasicAnalysisModule", "-13.614"))
self.ui.le_STMNEW_offset.setText(_translate("QWBasicAnalysisModule", "0"))
dlg_title = "提示"
str_info = "参数已全部重置！"
QMessageBox.information(self,dlg_title,str_info)
self.add_textBrowser_str(str_info)
#END
@pyqtSlot()
def on_btn_SaveResult_clicked(self):
pre_check = self.save_data_precheck()
if pre_check:
data_save_path = self.key_para["Data_Save_Path"]
self.creat_dir(data_save_path)
# ====================完成危险排查，开始数据保存！！！！============
distance, condutance, length, distance_draw, condutance_draw = self.distance, self.condutance, self.length, self.distance_draw, self.condutance_draw
self.save_analysis_data(distance_draw, condutance_draw, length, data_save_path)
self.save_fig(data_save_path)
self.save_goodtrace(distance_draw, condutance_draw, data_save_path)
self.save_singletrace(distance, condutance, data_save_path)
QMessageBox.information(self, "提示", f"所有数据均已保存，路径：{data_save_path}")
@pyqtSlot()
def on_btn_SelectDir_clicked(self):
"""
设置处理结果的保存目录
:return:
"""
desktop_path = get_desktop_path()
dlg_title = "选择处理结果的保存目录"
select_dir = QFileDialog.getExistingDirectory(self, dlg_title, desktop_path, QFileDialog.ShowDirsOnly)
if (select_dir != ""):
self.ui.le_Data_Save_Dir.setText(select_dir)
self.key_para[self.ui.le_Data_Save_Dir.objectName()] = select_dir
self.ui.btn_SaveResult.setEnabled(True)
@pyqtSlot()
def on_le_Jump_Gap_editingFinished(self):
"""
因为gap这个值很重要，所以用户在修改的时候需要加上提示
:return:
"""
result = QMessageBox.warning(self, "警告", "此参数的改变可能会导致切分结果的异常，请谨慎修改", QMessageBox.Ok | QMessageBox.Cancel,
QMessageBox.Cancel)
if result==QMessageBox.Cancel:
_translate = QtCore.QCoreApplication.translate
self.ui.le_Jump_Gap.setText(_translate("QWBasicAnalysisModule", "10000"))
# =============== 控件触发函数===============
def init_save_dir(self):
_translate = QtCore.QCoreApplication.translate
desktop_path=get_desktop_path()
self.ui.le_Data_Save_Dir.setText(_translate("QWBasicAnalysisModule", desktop_path))
def save_singletrace(self, distance, condutance, data_save_path):
# 思来想去，把数据保存为npy加快读取速度
ADDITIONAL_LENGTH = self.key_para["le_Additional_Length"]
single_trace_path = os.path.join(data_save_path, "Single_Trace")
self.creat_dir(single_trace_path)
# TODO
# 此处我建议保存additional参数，方便后续的分析 ->ok
np.savez(single_trace_path + "/single_trace.npz", distance_array=distance, condutance_array=condutance,additional_length=ADDITIONAL_LENGTH)
def save_goodtrace(self, distance, condutance, data_save_path):
trace_path = os.path.join(data_save_path, "Goodtrace")
self.creat_dir(trace_path)
data = np.array([distance, condutance]).T
np.savetxt(trace_path + "/goodtrace.txt", data, fmt='%.5e', delimiter='\t')
def save_fig(self, data_save_path):
img_path = os.path.join(data_save_path, "Images")
self.creat_dir(img_path)
self._2D_condutance_fig.fig.savefig(img_path + '/2D_Conductance.png', dpi=100, bbox_inches='tight')
self._1D_condutance_fig.fig.savefig(img_path + '/Conductance_count.png', dpi=100, bbox_inches='tight')
self._1D_length_fig.fig.savefig(img_path + '/Length_count.png', dpi=100, bbox_inches='tight')
def save_analysis_data(self, distance, condutance, length, data_save_path):
analysis_path = os.path.join(data_save_path, "Analysis")
self.creat_dir(analysis_path)
H_2D_cond, x_2D_edges, y_2D_edges = np.histogram2d(distance, condutance, bins=[500, 1000],
range=[[-0.5, 3], [-10, 1]])
H_1D_cond, bins_1D_edges = np.histogram(condutance, bins=1100, range=[-10, 1])
H_1D_length, bins_1D_length_edges = np.histogram(length, bins=100, range=[0, 3])
x_2D_edges_pad, y_2D_edges_pad = zero_pad(x_2D_edges, y_2D_edges)
np.savetxt(analysis_path + "/WA-BJ_3Dhist.txt", H_2D_cond * 2, fmt='%d', delimiter='\t')
hist_2D_scales = np.array([x_2D_edges_pad, y_2D_edges_pad]).T
np.savetxt(analysis_path + "/WA-BJ_3Dhist_scales.txt", hist_2D_scales, fmt='%.5e', delimiter='\t')
hist_1D_cond = np.array([bins_1D_edges[:-1], H_1D_cond]).T
np.savetxt(analysis_path + "/WA-BJ_logHist.txt", hist_1D_cond, fmt='%.5e', delimiter='\t')
hist_1D_length = np.array([bins_1D_length_edges[:-1], H_1D_length]).T
np.savetxt(analysis_path + "/WA-BJ_plateau.txt", hist_1D_length, fmt='%.5e', delimiter='\t')
def creat_dir(self, path):
try:
os.mkdir(str(path))
content = f"[{path}] 文件夹创建成功"
self.add_textBrowser_str(content)
self.add_statusBar_str(content)
except Exception as e:
print("创建文件夹目录时异常:", e)
def save_data_precheck(self):
"""
点击保存按钮之前的检查工作，包括数据是否存在和校验存储路径是否存在
:return: bool值
"""
if not self.key_para["SaveData_Statue"]:
QMessageBox.warning(self, "警告", "数据处理未完成，暂不能保存数据，请等待数据处理完成")
self.add_textBrowser_str("数据处理未完成，暂不能保存数据，请等待数据处理完成!!")
return False
dlg_title = "文件名设置"
txt_label = "请输入本次保存文件夹名"
default_name = "新建文件夹"
echo_mode = QLineEdit.Normal
# base_dir = self.key_para["le_Data_Save_Dir"]
base_dir=self.ui.le_Data_Save_Dir.text()
flag = False
while not flag:
text, OK = QInputDialog.getText(self, dlg_title, txt_label, echo_mode, default_name)
if OK:
save_path = os.path.join(base_dir, text)
IS_EXIST = os.path.exists(save_path)
if IS_EXIST:
QMessageBox.warning(self, "警告", "在该目录下文件名已存在或输入文件名不合法，请重新输入")
self.add_textBrowser_str("在该目录下文件名已存在或输入文件名不合法，请重新输入!!")
continue
flag = not flag
self.key_para["Data_Save_Path"] = save_path
return True
def check_savedir_statue(self):
"""
在程序初始化的时候检查存储路径的存在与否及其合法性
:return: bool值
"""
dir = self.ui.le_Data_Save_Dir.text()
if dir == "":
return False
else:
# 程序执行到这里，虽然路径不是""，但是应该检查路径的有效性
return os.path.exists(dir)
def set_load_state(self, file_list):
"""
设置文件加载状态
:param file_list: 调用FileDialog得到的文件列表
:return: Boolean
"""
if (len(file_list) > 0):
return True
return False
def add_textBrowser_str(self, content_str, showtime=True):
"""
在textBrowser中添加字符串
:param content_str: 字符串
:param showtime: 是否添加时间，默认true
:return: 无返回值
"""
if showtime:
current_time = get_current_time()
try:
self.ui.tbw_Log.append("[" + current_time + "] " + content_str)
except Exception as e:
print(f"发生异常：{e}")
else:
try:
self.ui.tbw_Log.append(content_str)
except Exception as e:
print(f"发生异常：{e}")
def add_textBrowser_list(self, content_list, showtime=True):
"""
在textBrowser中添加list
:param content_list: 字符串列表
:param showtime: 是否添加时间
:return: 无返回值
"""
if showtime:
current_time = get_current_time()
try:
self.ui.tbw_Log.append(current_time)
for content in content_list:
content = content.split("/")[-1]
self.ui.tbw_Log.append("-- " + content)
except Exception as e:
print(f"发生异常：{e}")
else:
try:
for content in content_list:
content = content.split("/")[-1]
self.ui.tbw_Log.append("-- " + content)
except Exception as e:
print(f"发生异常：{e}")
def add_statusBar_str(self, content_str):
"""
状态栏添加文字
:param content_str:字符串
:return:无
"""
try:
self.ui.statusbar.showMessage(":) " + content_str)
except Exception as e:
print(f"发生异常：{e}")
def set_progressBar_int(self, content_int):
"""
设置进度条数值
:param content_int: 进度条数值，整数
:return:无
"""
self.ui.progressBar.setValue(int(content_int))
def check_config(self):
if os.path.exists("config"):
dlg_title = "提示"
str_info = "检测到当前目录下存在历史配置文件，是否加载?"
reply = QMessageBox.question(self, dlg_title, str_info,
QMessageBox.Yes | QMessageBox.No,
QMessageBox.Yes)
if reply == QMessageBox.Yes:
self.get_last_para()
def get_panel_para(self):
"""
点击run按钮之后，获取到面板上的设置参数，将其返回 key_para
:return: dict，用来更新self.key_para
"""
"""
点击run按钮之后，获取到面板上的设置参数，将其保存在 key_para 中
:return: 无
"""
key_para = {}
try:
key_para["DEVICE_ID"] = self.ui.cmb_Device.currentIndex()
key_para["PROCESS"] = self.ui.cmb_Process.currentIndex()
key_para["SELECT_OPTION"] = self.ui.rdo_select_open.isChecked()
# 上面三个参数获取到对应的设备型号以及处理过程，筛选开关
le_obj_list = []
LINEEDIT_WIDGET_NEED_LIST = [self.ui.wdt_Basic_Set, self.ui.wdt_Select_Para]
for wdt in LINEEDIT_WIDGET_NEED_LIST:
le_obj_list.extend(self.get_same_widget(wdt, QLineEdit))
for obj in le_obj_list:
key_para[obj.objectName()] = float(obj.text())
# ===========================这两个位置比较尴=====手动添加
obj_list_manual = [self.ui.le_Additional_Length, self.ui.le_Data_Save_Dir]
key_para[obj_list_manual[0].objectName()] = int(obj_list_manual[0].text())
key_para[obj_list_manual[1].objectName()] = obj_list_manual[1].text()
# ==================下面的代码在后续添加了设备后需要手动添加！
key_para["DEVICE_0_PARA"] = self.get_device_para(self.ui.wdt_Device_0)
key_para["DEVICE_1_PARA"] = self.get_device_para(self.ui.wdt_Device_1)
key_para["DEVICE_2_PARA"] = self.get_device_para(self.ui.wdt_Device_2)
key_para["DEVICE_3_PARA"] = self.get_device_para(self.ui.wdt_Device_3)
except Exception as e:
print(f"发生异常：{e}")
return key_para
def get_last_para(self):
"""
加载程序同路径下保存好的历史参数并设置，
:return:
"""
_translate = QCoreApplication.translate
config = configparser.ConfigParser()
config.read('config', encoding='utf-8')
section_name = "PANEL_PARA"
LINEEDIT_WIDGET_NEED_LIST = [self.ui.wdt_Basic_Set, self.ui.wdt_Select_Para, self.ui.wdt_Device_0,
self.ui.wdt_Device_1, self.ui.wdt_Device_2, self.ui.wdt_Device_3]
le_obj_list = []
for wdt in LINEEDIT_WIDGET_NEED_LIST:
le_obj_list.extend(self.get_same_widget(wdt, QLineEdit))
for obj in le_obj_list:
obj.setText(_translate("QWBasicAnalysisModule", config.get(section_name, obj.objectName())))
obj_list_manual = [self.ui.le_Additional_Length, self.ui.le_Data_Save_Dir]
for obj in obj_list_manual:
obj.setText(_translate("QWBasicAnalysisModule", config.get(section_name, obj.objectName())))
self.add_textBrowser_str("已加载上次保存的参数...")
self.add_statusBar_str("已加载上次保存的参数...")
def save_config_para(self):
"""
完成关闭软件时面板参数的保存
:return: 无
"""
config = configparser.ConfigParser()
config.optionxform = str # 这一句相当的关键，因为config这个模块会把option自动的变为全小写，这个设置可以保持原样！
section_name = "PANEL_PARA"
config.add_section(section_name)
le_obj_list = []
LINEEDIT_WIDGET_NEED_LIST = [self.ui.wdt_Basic_Set, self.ui.wdt_Select_Para, self.ui.wdt_Device_0,
self.ui.wdt_Device_1, self.ui.wdt_Device_2, self.ui.wdt_Device_3]
for wdt in LINEEDIT_WIDGET_NEED_LIST:
le_obj_list.extend(self.get_same_widget(wdt, QLineEdit))
for obj in le_obj_list:
config.set(section_name, obj.objectName(), obj.text())
# ========这一部分需要手动添加=====
obj_list_manual = [self.ui.le_Additional_Length, self.ui.le_Data_Save_Dir]
for obj in obj_list_manual:
config.set(section_name, obj.objectName(), obj.text())
config.write(open('config', 'w'))
def get_same_widget(self, widget_name, activeX_name):
"""
获取某个 widget 中同类型的控件
:param widget_name: widget名，传入的是ui中的某个widget名
:param activeX_name: 控件类型，传入的是对象
:return: 寻找到的对象集合(List)
"""
return widget_name.findChildren(activeX_name)
def get_device_para(self, widget_name):
"""
获取不同设备对应的参数，注意只需要传入widget_name
:param widget_name:
:return: dict
"""
temp = {}
obj_list = self.get_same_widget(widget_name, QLineEdit)
for obj in obj_list:
temp[obj.objectName()] = float(obj.text())
return temp
def split_filelist(self, file_list):
"""
遗憾的是，还是单个文件比较快，此功能dead
开发此功能的目的是加载多个文件之后，因为采用多进程的进程池去处理，每次处理一个文件总是感觉会有点浪费开销
所以，可以吧filelist做一个变化，几个文件为一组，变成二维数组
:param file_list:
:return: 修改好的filelist
"""
file_list_new = []
GROUP_SIZE = 3 # 一次处理两个文件----------经过测试，同时处理三个文件效率较高！！
len_file_list = len(file_list)
if (len(file_list) <= GROUP_SIZE):
return file_list_new.append(file_list)
len_new_list = len_file_list // GROUP_SIZE if len_file_list % GROUP_SIZE == 0 else len_file_list // GROUP_SIZE + 1
for i in range(len_new_list):
file_list_new.append(file_list[i * GROUP_SIZE:GROUP_SIZE * (i + 1)])
return file_list_new
def draw_prepare(self):
"""
绘图前的准备
:return: 无
"""
print('计算进程安全退出，开始计算绘图数据..........')
original_dataset = self.data_analysis.dataset
# 这里的这个返回值不管是单个文件，还是多个文件，都是List
self.dataset,statue = self.get_NOTNULL_dataset(original_dataset)
if not statue:
self.set_progressBar_int(0)
self.ui.actRun.setEnabled(True)
# TODO
# 程序运行到此处，说明一条都没切出来，需要修改切分范围，所以可以删掉一些东西
del self.dataset
del self.data_analysis
else:
self.cal_draw() # 检查通过之后，开始计算绘图所需数据
def stop_thread(self, thread):
"""
多进程中进程的停止
:param thread: 需传入对应的进程
:return: 无返回值
"""
thread.quit()
thread.wait()
print(f"退出{thread.currentThread()}线程，现在线程状态 ：", thread.isRunning())
def check_dataset(self, dataset):
"""
检查切分数据是否成功
:param dataset: 单个的dataset
:return: bool
"""
zero_index = dataset[2]
if (len(zero_index) < 2):
return False
return True
def get_NOTNULL_dataset(self,original_dataset):
"""
因为存在多进程的问题，有可能那么几组就一条也没有，后面的计算绘图数据可能会有问题，以及
后面多组数据的叠加也会有问题，就头疼哈哈，所以这里直接把空的去掉！
:param original_dataset: 多进程得到的数据
:return: 非空数据
"""
effective_counts=0
dataset=[]
for data in original_dataset:
if self.check_dataset(data):
dataset.append(data)
effective_counts+=1
else:
continue
if effective_counts == 0:
QMessageBox.critical(self, "错误", "切分失败，建议如下：\n1.调整HIGH_CUT\n2.修改仪器拟合参数\n3.数据质量太差")
return dataset,False
else:
return dataset,True
def cal_draw(self):
"""
开启计算绘图数据的进程
:return:无
"""
self._drawdata_thread = QThread()
self.cal_draw_data = CalDrawData(self.key_para, self.dataset)
self.cal_draw_data.pbar.connect(self.set_progressBar_int)
self.cal_draw_data.tbw.connect(self.add_textBrowser_str)
self.cal_draw_data.sbar.connect(self.add_statusBar_str)
self.cal_draw_data.run_end.connect(lambda: self.stop_thread(self._drawdata_thread))
self.cal_draw_data.moveToThread(self._drawdata_thread)
self._drawdata_thread.started.connect(self.cal_draw_data.run)
self._drawdata_thread.finished.connect(self.start_draw)
show_info = "绘图所需数据计算中..."
self.add_statusBar_str(show_info)
self.add_textBrowser_str(show_info)
self._drawdata_thread.start()
print(f'开启{self._drawdata_thread.currentThread()}线程，现在线程状态 ：', self._drawdata_thread.isRunning())
def start_draw(self):
"""
准备绘图及相关按钮设置
:return: 无
"""
print('绘图数据计算完成，准备绘图..........')
self.set_progressBar_int(96)
self.draw_dataset = self.cal_draw_data.draw_dataset
draw_dataset = copy.deepcopy(self.draw_dataset)
self.distance, self.condutance, self.length, self.distance_draw, self.condutance_draw, ALL_TRACE_NUM, SELECT_TRACE_NUM = self.get_cumulative_drawdata(
draw_dataset)
self.set_trace_ratio(ALL_TRACE_NUM, SELECT_TRACE_NUM)
show_info = "开始绘图..."
self.add_statusBar_str(show_info)
self.add_textBrowser_str(show_info)
self.draw(self.distance_draw, self.condutance_draw, self.length)
self.set_progressBar_int(100)
self.ui.btn_Update.setEnabled(True)
self.ui.btn_Redraw.setEnabled(True)
def draw(self, distance, condutance, length):
"""
画图
:param distance: 距离
:param condutance: 叠加后的一维电导
:param length: 叠加后的一维长度
:return: 无
"""
if self.DRAW_FIRST:
self.DRAW_FIRST = False
else:
self._2D_condutance_layout.removeWidget(self._2D_condutance_fig)
self._2D_condutance_layout.removeWidget(self._2D_navi_bar)
self._1D_condutance_layout.removeWidget(self._1D_condutance_fig)
self._1D_condutance_layout.removeWidget(self._1D_navi_bar)
self._1D_length_layout.removeWidget(self._1D_length_fig)
self._1D_length_layout.removeWidget(self._length_navi_bar)
sip.delete(self._2D_condutance_fig)
sip.delete(self._2D_navi_bar)
sip.delete(self._1D_condutance_fig)
sip.delete(self._1D_navi_bar)
sip.delete(self._1D_length_fig)
sip.delete(self._length_navi_bar)
plt.close("all")
FONTSIZE = '8'
CM = plt.cm.coolwarm
_2D_BINSX = int(self.key_para["le_2D_BinsX"])
_2D_BINSY = int(self.key_para["le_2D_BinsY"])
_2D_XLEFT = self.key_para["le_2D_Xleft"]
_2D_XRIGHT = self.key_para["le_2D_Xright"]
_2D_YLEFT = self.key_para["le_2D_Yleft"]
_2D_YRIGHT = self.key_para["le_2D_Yright"]
_1D_COND_XLEFT = self.key_para["le_1D_Cond_Xleft"]
_1D_COND_XRIGHT = self.key_para["le_1D_Cond_Xright"]
_1D_COND_BINS = int(self.key_para["le_1D_Cond_Bins"])
_1D_LENG_XLEFT = self.key_para["le_1D_Leng_Xleft"]
_1D_LENG_XRIGHT = self.key_para["le_1D_Leng_Xright"]
_1D_LENG_BINS = int(self.key_para["le_1D_Leng_Bins"])
_mean_length = round(np.mean(length), 2)
_sigma_length = np.std(length)
self._2D_condutance_fig = MyFigureCanvas()
*_2D_DATA_FIG, image = self._2D_condutance_fig.axes.hist2d(distance, condutance, bins=[_2D_BINSX, _2D_BINSY],
range=[[_2D_XLEFT, _2D_XRIGHT],
[_2D_YLEFT, _2D_YRIGHT]], density=1, vmin=0,
vmax=1,cmap=CM)
self._2D_condutance_fig.axes.set_xlabel('Length / nm', fontsize=FONTSIZE)
self._2D_condutance_fig.axes.set_ylabel('Conductance', fontsize=FONTSIZE)
# self._2D_condutance_fig.fig.tight_layout()
self._2D_condutance_fig.fig.colorbar(image, pad=0.02, aspect=50,ticks=None)
self._2D_navi_bar = MyNavigationToolbar(self._2D_condutance_fig, self._2D_condutance_fig.main_frame)
self._2D_navi_bar.actions()
self._2D_condutance_layout.addWidget(self._2D_condutance_fig)
self._2D_condutance_layout.addWidget(self._2D_navi_bar)
self.ui.grp_2D_Cloud.setLayout(self._2D_condutance_layout)
self._1D_condutance_fig = MyFigureCanvas()
_1D_DATA_FIG = self._1D_condutance_fig.axes.hist(condutance, bins=_1D_COND_BINS, color='green', alpha=0.8,
range=[_1D_COND_XLEFT, _1D_COND_XRIGHT])
self._1D_condutance_fig.axes.set_xlabel('Conductance', fontsize=FONTSIZE)
self._1D_condutance_fig.axes.set_ylabel('Counts', fontsize=FONTSIZE)
self._1D_condutance_fig.axes.set_xlim((_1D_COND_XLEFT, _1D_COND_XRIGHT))
self._1D_condutance_fig.axes.grid(True)
# self._1D_condutance_fig.axes.yaxis.get_major_formatter().set_powerlimits((0, 1))
# 换一种科学计数法
self._1D_condutance_fig.axes.ticklabel_format(style='scientific',scilimits=(0,2),useMathText=True)
# scilimits=(m,n)表示如果刻度范围超出10^m10m到10^n10n，那么就是用科学计数法。
# 如果将scilimits参数设为(0,0)，那么对于所有的刻度范围都自动显示成科学计数的形式。所以这里设置为10^2以外的才用科学计数法
# 令useMathText=False的时候，会显示为1eX1eX的形式，useMathText=True的时候，会显示成10^X10X的形式。
# self._1D_condutance_fig.fig.tight_layout()
# TODO 这个后面考虑是采用自己继承的NavigationToolbar还是tight_layout()？
self._1D_navi_bar = MyNavigationToolbar(self._1D_condutance_fig, self._1D_condutance_fig.main_frame)
self._1D_condutance_layout.addWidget(self._1D_condutance_fig)
self._1D_condutance_layout.addWidget(self._1D_navi_bar)
self.ui.grp_1D_Condutance.setLayout(self._1D_condutance_layout)
self._1D_length_fig = MyFigureCanvas()
_, BINS_LENGTH, _ = self._1D_length_fig.axes.hist(length, density=True, bins=_1D_LENG_BINS,
range=[_1D_LENG_XLEFT, _1D_LENG_XRIGHT])
# self._1D_length_fig.axes.yaxis.get_major_formatter().set_powerlimits((0, 1))
self._1D_length_fig.axes.ticklabel_format(style='scientific',scilimits=(0,2),useMathText=True)
temp_y = norm.pdf(BINS_LENGTH, _mean_length, _sigma_length)
self._1D_length_fig.axes.plot(BINS_LENGTH, temp_y, "r--", label="length: " + str(_mean_length))
self._1D_length_fig.axes.set_xlabel('Length / nm', fontsize=FONTSIZE)
self._1D_length_fig.axes.set_ylabel('counts', fontsize=FONTSIZE)
self._1D_length_fig.axes.set_xlim((_1D_LENG_XLEFT, _1D_LENG_XRIGHT))
self._1D_length_fig.axes.grid(True)
self._1D_length_fig.axes.legend(loc=1)
# self._1D_length_fig.fig.tight_layout()
self._length_navi_bar = MyNavigationToolbar(self._1D_length_fig, self._1D_length_fig.main_frame)
self._1D_length_layout.addWidget(self._1D_length_fig)
self._1D_length_layout.addWidget(self._length_navi_bar)
self.ui.grp_1D_Length.setLayout(self._1D_length_layout)
self.key_para["SaveData_Statue"] = True # 这个true放在这里的目的是只要绘图完成一遍，就说明产生了新数据，可以保存
def set_trace_ratio(self, ALL_TRACE_NUM, SELECT_TRACE_NUM):
"""
在ui上面设置筛选结果
:param ALL_TRACE_NUM: 所有曲线条数
:param SELECT_TRACE_NUM: 筛选曲线条数
:return: 无
"""
SELECT_RATIO = SELECT_TRACE_NUM / ALL_TRACE_NUM * 100
self.ui.le_AllTrace.setText(str(ALL_TRACE_NUM))
self.ui.le_SelectedTrace.setText(str(SELECT_TRACE_NUM))
self.ui.le_Ratio.setText(str(SELECT_RATIO))
def get_cumulative_drawdata(self, draw_dataset):
"""
得到聚合结果（将多进程得到的绘图数据聚合）
:param draw_dataset: 绘图数据（list，list长度为绘图进程数）
:return: 返回聚合结果
"""
if len(draw_dataset) == 1:
return draw_dataset[0][0], draw_dataset[0][1], draw_dataset[0][2], draw_dataset[0][3], draw_dataset[0][4], \
draw_dataset[0][5], draw_dataset[0][6]
distance, condutance, length, distance_draw, condutance_draw, ALL_TRACE_NUM, SELECT_TRACE_NUM = draw_dataset[0][
0], \
draw_dataset[0][
1], \
draw_dataset[0][
2], \
draw_dataset[0][
3], \
draw_dataset[0][
4], \
draw_dataset[0][
5], \
draw_dataset[0][
6]
for draw_data in draw_dataset[1:]:
distance = np.concatenate((distance, draw_data[0]))
condutance = np.concatenate((condutance, draw_data[1]))
length = np.concatenate((length, draw_data[2]))
distance_draw = np.concatenate((distance_draw, draw_data[3]))
condutance_draw = np.concatenate((condutance_draw, draw_data[4]))
ALL_TRACE_NUM += draw_data[5]
SELECT_TRACE_NUM += draw_data[6]
return distance, condutance, length, distance_draw, condutance_draw, ALL_TRACE_NUM, SELECT_TRACE_NUM
def __del__(self):
print("QmyBasicAnalysisModule 对象被删除，基础分析模块结束")
from spotify_client import SpotifyClient
# from looking at the KDE plots in plots.py, the darkest regions were in these bounds:
max_energy = 0.8
min_energy = 0.6
max_valence = 0.63
min_valence = 0.47
max_tempo = 133
min_tempo = 77
new_songs = []
# now we want to go through our dataframe of liked songs and get our "ideal songs":
# songs which fall into the bounds of the KDE plot highest density regions.
name = "audio_features.csv"
ideal_songs = DataAnalysis.ideal_song(pd, name, max_energy, max_valence,
max_tempo, min_energy, min_valence,
min_tempo)
print("Obtained ideal songs to generate your recommendations successfully.")
# now, using this list of ideal songs, we need to get our list of recommendations:
# we will loop through so that we can get recommendations from all of our ideal songs:
SpotifyClient.recommended_songs(sp, ideal_songs, new_songs, max_energy,
max_valence, max_tempo, min_energy,
min_valence, min_tempo)
print("Found recommendations for you successfully.")
# create the playlist to put our recommended songs into
playlist = SpotifyClient.create_playlist(sp)
# add the recommended songs to our new playlist
SpotifyClient.add_songs(sp, playlist, new_songs)
for song_id in song_ids:
SpotifyClient.get_audio_features(sp, song_id, danceability, energy,
valence, tempo)
print("Obtained audio features of your songs successfully.")
# next, to do the data analysis with DataFrames, we need to convert our separate lists into a list of lists:
audio_features = []
audio_features.append(danceability)
audio_features.append(energy)
audio_features.append(valence)
audio_features.append(tempo)
audio_features.append(song_ids)
audio_features.append(dates_added)
# convert the audio_features list into a csv file
df = DataAnalysis.dataframe_conversion(pd, audio_features)
print("Created file 'audio_features.csv' successfully.")
# look at user's 50 most recently played songs
SpotifyClient.recently_played(sp, song_ids, in_liked_songs,
recently_played_songs)
print("Obtained your 50 most recently played songs successfully.")
# now lets get the audio features for our recently played songs:
for recent_song in recently_played_songs:
SpotifyClient.get_audio_features(sp, recent_song, danceability_recent,
energy_recent, valence_recent,
tempo_recent)
print(
"Obtained the audio features of your recently played songs successfully.")
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("config", help="The JSON config file to load.")
args = parser.parse_args()
config = {}
with open(args.config, 'r') as f:
config = json.load(f)
weekday_only = config.get('weekdays-only', False)
# fail hard if not found
db_url = config['database-url']
da = DataAnalysis(db_url)
outcomes = ['success']
weekdays_only = config['weekdays-only']
def display(data):
pre = data.branch
if pre is None:
pre = ''
days = sorted(data.data.keys())
for i in days:
print("{0}>{1}:{2}".format(pre, i,data.data[i]))
#print("{0}-----{1}".format(pre, data.average))
print("{0}-----{1}".format(pre, len(data.data)))
def AnalysisTrainModels():
if request.method == 'GET':
dataset_name = 'data.csv'
out = DataAnalysis(dataset_name)
train = MainML(dataset_name)
return train
with open(sys.argv[1], 'r') as f:
config = json.load(f)
if not config.has_key('repos'):
print("Error parsing config file! Missing 'repos' stanza!")
print(config)
sys.exit(1)
if not config.has_key('database-url'):
print("Error parsing config file! Missing 'database-url'!")
print(config)
sys.exit(1)
weekdays_only = config.get('weekdays-only', True)
da = DataAnalysis(config['database-url'])
for repo in config['repos']:
page_title = repo.get('title', '').format(date=datetime.datetime.now())
hash_id = hashlib.md5(repo['path']).hexdigest()
data = []
for chart in config['charts']:
num_days = chart['duration']
d = []
if chart['data-type'] == 'average':
# Compute averages
for branch in repo['highlight-branches']:
d.append(da.compute_averages(hash_id, num_days, ['success'],
class Signal(QtWidgets.QWidget):
signal = QtCore.pyqtSignal()
class COMThreadManual(QtCore.QThread):
def __init__(self):
QtCore.QThread.__init__(self)
def __del__(self):
self.wait()
def run(self):
ui.refresh_com()
if __name__ == "__main__":
import sys
app = QtWidgets.QApplication(sys.argv)
app.setStyle('Fusion')
pinchSensorUI = MainWindow()
ui = Ui_pinchSensorUI()
ui2 = DataAnalysis()
pinchSensorUI.show()
sys.exit(app.exec_())
# TODO bootup loading screen