def generateMeasuredFrames(self, accelerator):
if (accelerator == 'SR'):
typesOfMagnets = [
'quadrupole', 'dipole-B1', 'dipole-B2', 'dipole-BC'
]
for magnetType in typesOfMagnets:
DataUtils.generateMeasuredFrames(self.app.entitiesDictMeas[accelerator], self.app.entitiesDictNom[accelerator],\
self.app.frameDict[accelerator], 'SR', self, typeOfMagnets=magnetType, isTypeOfMagnetsIgnored=False)
return
if (accelerator == 'booster'):
DataUtils.generateMeasuredFrames(self.app.entitiesDictMeas[accelerator], self.app.entitiesDictNom[accelerator],\
self.app.frameDict[accelerator], 'booster', self)
return
if (accelerator == 'LTB'):
DataUtils.generateMeasuredFrames(self.app.entitiesDictMeas[accelerator], self.app.entitiesDictNom[accelerator],\
self.app.frameDict[accelerator], 'LTB', self)
return
if (accelerator == 'BTS'):
DataUtils.generateMeasuredFrames(self.app.entitiesDictMeas[accelerator], self.app.entitiesDictNom[accelerator],\
self.app.frameDict[accelerator], 'BTS', self)
return
if (accelerator == 'FE'):
DataUtils.generateMeasuredFrames(self.app.entitiesDictMeas[accelerator], self.app.entitiesDictNom[accelerator],\
self.app.frameDict[accelerator], 'FE', self)
return
def plotAbsolute(self, accelerator):
self.logMessage("Plotando o perfil de alinhamento absoluto do " +
accelerator + "...")
self.app.processEvents()
# checando se todos os arquivos foram devidamente carregados
if (not self.app.isNominalsLoaded[accelerator]
or not self.app.isMeasuredLoaded[accelerator]
or not self.app.isLookuptableLoaded[accelerator]):
self.logMessage(
"Erro ao plotar gráfico: nem todos os arquivos foram carregados"
)
return
magnetsDeviations = DataUtils.calculateMagnetsDeviations(
self.app.frameDict[accelerator])
plotArgs = {'accelerator': accelerator, 'filtering':\
{'SR': {'allDofs': self.plotAllDofs_SR.isChecked(), 'lineWithinGirder': self.lineWithinGirder_SR.isChecked(), 'reportMode': self.reportMode_SR.isChecked(), 'errorbar': self.errorbar_SR.isChecked()},\
'booster': {'quad02': self.plotQuad02_booster.isChecked(), 'lineWithinGirder': False, 'reportMode': self.reportMode_booster.isChecked(), 'errorbar': self.errorbar_booster.isChecked()},\
'LTB': {'lineWithinGirder': False, 'reportMode': False, 'errorbar': self.errorbar_LTB.isChecked()},\
'BTS': {'lineWithinGirder': False, 'reportMode': False, 'errorbar': self.errorbar_BTS.isChecked()},\
'FE': {'lineWithinGirder': False, 'reportMode': False, 'errorbar': self.errorbar_FE.isChecked()}}}
DataUtils.plotDevitationData(magnetsDeviations, **plotArgs)
def loadLookuptableFile(self, accelerator, filePath=None):
if (filePath == ""):
return
filepath = filePath or self.openFileNameDialog('lookup de frames',
accelerator)
if (not filepath):
return
fileName = filepath.split('/')[len(filepath.split('/')) - 1]
fileExtension = fileName.split('.')[1]
if (fileExtension == 'txt' or fileExtension == 'csv'
or fileExtension == 'TXT' or fileExtension == 'CSV'):
lookuptable = DataUtils.readCSV(filepath, 'lookuptable')
elif (fileExtension == 'xlsx'):
lookuptable = DataUtils.readExcel(filepath, 'lookuptable')
else:
self.logMessage("Format of file " + fileName + "not supported.")
return
self.app.lookupTable[accelerator] = lookuptable
self.app.isLookuptableLoaded[accelerator] = True
if (accelerator == 'SR'):
self.ledFrames_SR.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileFrames_SR.setText(fileName)
self.fileFrames_SR.setToolTip(fileName)
elif (accelerator == 'booster'):
self.ledFrames_booster.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileFrames_booster.setText(fileName)
self.fileFrames_booster.setToolTip(fileName)
elif (accelerator == 'LTB'):
self.ledFrames_LTB.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileFrames_LTB.setText(fileName)
self.fileFrames_LTB.setToolTip(fileName)
elif (accelerator == 'BTS'):
self.ledFrames_BTS.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileFrames_BTS.setText(fileName)
self.fileFrames_BTS.setToolTip(fileName)
elif (accelerator == 'FE'):
self.ledFrames_FE.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileFrames_FE.setText(fileName)
self.fileFrames_FE.setToolTip(fileName)
self.app.lookupFileName[accelerator] = filepath
self.logMessage("Lookuptable de frames do " + accelerator +
" carregada.")
self.checkAllFilesAndProcessData(accelerator)
def runDebugFunctions(self):
self.loadEnv()
# creating frame's dict for the particular accelerator
self.app.frameDict['SR'] = DataUtils.generateFrameDict(
self.app.lookupTable['SR'])
entitiesDictNom = SR.createObjectsStructure(self.app.nominals['SR'])
self.app.entitiesDictNom['SR'] = entitiesDictNom
DataUtils.debug_transformToLocalFrame(self.app.entitiesDictNom['SR'],
self.app.frameDict['SR'], 'SR')
DataUtils.debug_translatePoints(self.app.entitiesDictNom['SR'],
self.app.frameDict['SR'])
def plotRelative(self, accelerator):
self.logMessage("Plotando o perfil de alinhamento relativo do " +
accelerator + "...")
self.app.processEvents()
# checando se todos os arquivos foram devidamente carregados
if (not self.app.isNominalsLoaded[accelerator]
or not self.app.isMeasuredLoaded[accelerator]
or not self.app.isLookuptableLoaded[accelerator]):
self.logMessage(
"Erro ao plotar gráfico: nem todos os arquivos foram carregados"
)
return
magnetsDeviations = DataUtils.calculateMagnetsRelativeDeviations(
self.app.frameDict[accelerator])
# DataUtils.writeToExcel('../data/output/sr-devs.xlsx', magnetsDeviations)
DataUtils.plotRelativeDevitationData(magnetsDeviations, accelerator)
def __predict(self, model, x):
"""
prediction
Args:
-----
model: keras.model
x: str
return:
np.array(float/int)
"""
x_array = self.__reshapeX(x)
x_numeric = DataUtils.numeric_encoding(x_array, self.UNIFORM_LENGTH,
self.word_map)
return model.predict(x_numeric)
def __init__(self):
"""
model constructor with preloaded features
"""
# find the model path
self.model_abs = self.__load_model_weight('model_smiles_cnn.json',\
'weights.best.hdf5')
self.model_ems = self.__load_model_weight('model_smiles_ems_dropna.json',\
'ems_dropna.best1.hdf5')
self.model_epsilon = self.__load_model_weight('model_smiles_epsilon_lstm.json',\
'epsilon.best_-1.hdf5')
self.model_qy = self.__load_model_weight('model_lstm_qy.json', \
'weights.qy_best.hdf5')
self.word_map = DataUtils.load_wordmap_from_json(
os.path.join(MODEL_PATH, 'smiles_wordmap.json'))
def plotComparativeDev(self):
self.logMessage(
"Plotando o perfil de alinhamento de todo os aceleradores...")
self.app.processEvents()
magnetsDeviations_LTB = DataUtils.calculateMagnetsDeviations(
self.app.frameDict['LTB'])
magnetsDeviations_booster = DataUtils.calculateMagnetsDeviations(
self.app.frameDict['booster'])
magnetsDeviations_BTS = DataUtils.calculateMagnetsDeviations(
self.app.frameDict['BTS'])
magnetsDeviations_SR = DataUtils.calculateMagnetsDeviations(
self.app.frameDict['SR'])
deviation = [
magnetsDeviations_LTB, magnetsDeviations_booster,
magnetsDeviations_BTS, magnetsDeviations_SR
]
sizes = [
magnetsDeviations_LTB.iloc[:, 0].size,
magnetsDeviations_booster.iloc[:, 0].size,
magnetsDeviations_BTS.iloc[:, 0].size
]
lenghts = [
sizes[0], sizes[1] + sizes[0], sizes[2] + sizes[1] + sizes[0]
]
deviations = pd.concat(deviation)
plotArgs = {'accelerator': 'SR', 'len': lenghts, 'filtering':\
{'SR': {'allDofs': self.plotAllDofs_SR.isChecked(), 'lineWithinGirder': self.lineWithinGirder_SR.isChecked(), 'reportMode': self.reportMode_SR.isChecked(), 'errorbar': self.errorbar_SR.isChecked()},\
'booster': {'quad02': self.plotQuad02_booster.isChecked(), 'lineWithinGirder': False, 'reportMode': self.reportMode_booster.isChecked(), 'errorbar': self.errorbar_booster.isChecked()},\
'LTB': {'lineWithinGirder': False, 'reportMode': False, 'errorbar': self.errorbar_LTB.isChecked()},\
'BTS': {'lineWithinGirder': False, 'reportMode': False, 'errorbar': self.errorbar_BTS.isChecked()}}}
DataUtils.plotComparativeDeviation(deviations, **plotArgs)
import concurrent.futures
from tabulate import tabulate
from timeit import default_timer as timer
from dataUtils import DataUtils
from dataAggregator import DataAggregator
from baselines import MostPopularRecommender, MostRecentRecommender, MeanScoreRecommender
from evalMethods import evaluate_recall, evaluate_arhr, evaluate_mse
from contentbased import ContentBasedRecommender
if __name__ == '__main__':
DATANUM = 0 # number of files, 0 to load all
K = 10
dataHelper = DataUtils()
print("loading data...")
entries = dataHelper.load_data("active1000", num=DATANUM)
numLoaded = len(entries)
print(f"loaded {numLoaded} events.")
# Filter out unhelpful rows
print("filtering data...")
filtered_data = dataHelper.filter_data(entries)
numLost = numLoaded - len(filtered_data)
print(
f"filtered to {len(filtered_data)} events. {numLost} events ({numLost / numLoaded:%}) discarded."
)
# Re-index document and user IDs to start at 0 and be sequential, manually
#print("re-indexing data...")
def processInternalData(self, accelerator):
# creating frame's dict for the particular accelerator
self.app.frameDict[accelerator] = DataUtils.generateFrameDict(
self.app.lookupTable[accelerator])
if (accelerator == 'booster'):
entitiesDictNom = Booster.createObjectsStructure(
self.app.nominals['booster'])
entitiesDictMeas = Booster.createObjectsStructure(
self.app.measured['booster'])
elif (accelerator == 'SR'):
entitiesDictNom = SR.createObjectsStructure(
self.app.nominals['SR'])
entitiesDictMeas = SR.createObjectsStructure(
self.app.measured['SR'], shiftsB1=self.app.shiftsB1)
elif (accelerator == 'LTB'):
entitiesDictNom = LTB.createObjectsStructure(
self.app.nominals['LTB'])
entitiesDictMeas = LTB.createObjectsStructure(
self.app.measured['LTB'])
elif (accelerator == 'BTS'):
entitiesDictNom = BTS.createObjectsStructure(
self.app.nominals['BTS'])
entitiesDictMeas = BTS.createObjectsStructure(
self.app.measured['BTS'])
elif (accelerator == 'FE'):
entitiesDictNom = FE.createObjectsStructure(
self.app.nominals['FE'])
entitiesDictMeas = FE.createObjectsStructure(
self.app.measured['FE'])
self.app.entitiesDictNom[accelerator] = entitiesDictNom
self.app.entitiesDictMeas[accelerator] = entitiesDictMeas
DataUtils.transformToLocalFrame(self.app.entitiesDictNom[accelerator],
self.app.frameDict[accelerator],
accelerator)
DataUtils.transformToLocalFrame(self.app.entitiesDictMeas[accelerator],
self.app.frameDict[accelerator],
accelerator)
self.generateMeasuredFrames(accelerator)
self.app.frameDict[
accelerator] = DataUtils.sortFrameDictByBeamTrajectory(
self.app.frameDict[accelerator], accelerator)
self.app.report[accelerator] = DataUtils.generateReport(
self.app.frameDict[accelerator], accelerator)
# habilita botões de plot
if (accelerator == 'SR'):
self.plotAbsolute_SR.setEnabled(True)
self.expAbsolute_SR.setEnabled(True)
self.expMachModel_SR.setEnabled(True)
self.reportProgress_SR.setEnabled(True)
elif (accelerator == 'booster'):
self.plotAbsolute_booster.setEnabled(True)
self.expAbsolute_booster.setEnabled(True)
self.reportProgress_booster.setEnabled(True)
elif (accelerator == 'LTB'):
self.plotAbsolute_LTB.setEnabled(True)
self.expAbsolute_LTB.setEnabled(True)
elif (accelerator == 'BTS'):
self.plotAbsolute_BTS.setEnabled(True)
self.expAbsolute_BTS.setEnabled(True)
elif (accelerator == 'FE'):
self.plotAbsolute_FE.setEnabled(True)
self.expAbsolute_FE.setEnabled(True)
def loadMeasuredFile(self, accelerator, filePath=None):
if (filePath == ""):
return
filepath = filePath or self.openFileNameDialog('pontos medidos',
accelerator)
if (not filepath):
return
fileName = filepath.split('/')[len(filepath.split('/')) - 1]
fileExtension = fileName.split('.')[1]
if (fileExtension == 'txt' or fileExtension == 'csv'
or fileExtension == 'TXT' or fileExtension == 'CSV'):
measured = DataUtils.readCSV(filepath, 'points')
elif (fileExtension == 'xlsx'):
measured = DataUtils.readExcel(filepath, 'points')
else:
self.logMessage("Formato do arquivo " + fileName +
" não suportado.",
severity="alert")
return
self.app.measured[accelerator] = measured
self.app.isMeasuredLoaded[accelerator] = True
if (accelerator == 'SR'):
self.ledMeas_SR.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileMeas_SR.setText(fileName)
self.fileMeas_SR.setToolTip(fileName)
elif (accelerator == 'booster'):
self.ledMeas_booster.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileMeas_booster.setText(fileName)
self.fileMeas_booster.setToolTip(fileName)
elif (accelerator == 'LTB'):
self.ledMeas_LTB.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileMeas_LTB.setText(fileName)
self.fileMeas_LTB.setToolTip(fileName)
elif (accelerator == 'BTS'):
self.ledMeas_BTS.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileMeas_BTS.setText(fileName)
self.fileMeas_BTS.setToolTip(fileName)
elif (accelerator == 'FE'):
self.ledMeas_FE.setStyleSheet(
"background-color:green; border-radius:7px;")
self.fileMeas_FE.setText(fileName)
self.fileMeas_FE.setToolTip(fileName)
self.app.ptsMeasFileName[accelerator] = filepath
self.logMessage("Arquivo com medidos do " + accelerator +
" carregado.")
self.checkAllFilesAndProcessData(accelerator)
""" @version: 1.0 @author: Jie Lin @Mail: [email protected] @file: dataUtils.py @time: 09/27/2018 4:08pm @purpose: this files contain a main method to call each function in class DataUtils in order to clean data @code environment: ubuntu 18.01 """ from dataUtils import DataUtils # the main method to call each function in Datautils class if __name__ == '__main__': # to call a DataUtils class dataUtil = DataUtils() # pass data address to class to get a dataframe messyData = dataUtil.readFiles('./SIIM2016_Messy_Fake_EMRdata.csv') print(messyData) #print data under resource column and show without duplicate with the number of each kind of data print("showing data under resource:") print(dataUtil.printCDataWoutDu(messyData, "resource")) print() #select TRCT1 under resource column print("showing data with resource name = \'TRCT1\'") resourceTRCT1 = dataUtil.findDataUnCol(messyData, "TRCT1", "resource") print(resourceTRCT1) print()