def testClustering(self):
Plugin.setEnviron()
volList = self._getVolList()
self._getAverageVol(volList)
dictNames = {}
groupDict = {}
prot = Prot3DAutoClassifier(classMethod=1)
print("Mehod: ", prot.classMethod.get())
# matrix = self._estimatePCA(volList)
matrix, _ = self._mrcToNp(volList)
labels = prot._clusteringData(matrix)
if labels is not None:
f = open('method_%s.txt' % 1, 'w')
for vol, label in zip(volList, labels):
dictNames[vol] = label
for key, value in sorted(dictNames.items()):
groupDict.setdefault(value, []).append(key)
for key, value in groupDict.items():
line = '%s %s\n' % (key, value)
f.write(line)
f.close()
print(labels)
def convertInputStep(self, resetDeps):
""" Preprocess all volumes prior to clustering"""
Plugin.setEnviron()
self._convertVolumes()
pxSize = self.inputVolumes.get().getSamplingRate()
listVol = self._getPathMaps()
lowPass = self.lowPass.get()
lowRaised = self.lowRaised.get()
for volFn in listVol:
inputMap = volFn + ':mrc'
outMap = volFn.split('.')[0] + '_filtered.mrc:mrc'
dictParam = {'-i': inputMap,
'-o': outMap,
'--sampling': pxSize
}
args = ' '.join('%s %s' %(k,v) for k,v in dictParam.items())
args += ' --fourier low_pass %s %s' %(lowPass, lowRaised)
self.runJob('xmipp_transform_filter', args)
print("Saving average volume")
self._saveAverageVol()
if self.alignVolumes:
self._alignVolumes()
def __init__(self, datafiles, norm_file):
super(LoaderPredict, self).__init__()
Plugin.setEnviron()
self.normalization = Normalization(None)
self.normalization.load(norm_file)
self._data = [i for i in datafiles]
def __init__(self, filename_list, size=(1, 419, 419)):
super(LoaderPredict, self).__init__()
Plugin.setEnviron()
self._data = []
self._size = size
self._data += [{'file': i} for i in filename_list]
def testCorrectAnisotrophy(self):
Plugin.setEnviron()
npIh = NumpyImgHandler()
volNp = npIh.loadMrc(self.volume1)
vol, volFt = correctAnisotropy(volNp, 0.5, 0.55, 0.1, 0.45)
print("shape: ", vol.shape)
npIh.saveMrc(vol.astype(volNp.dtype), "corrected.mrc")
npIh.saveMrc(volFt.astype(volNp.dtype), "corrected_ft.mrc")
def evaluationStep(self):
Plugin.setEnviron()
print('Starting evaluation step')
print('which level: ', self._level)
self._copyLevelMaps()
self._evalStop()
self._mergeMetaDatas()
print('Finishing evaluation step')
def __init__(self, data, norm_file):
super(LoaderTrain, self).__init__()
Plugin.setEnviron()
self.normalization = Normalization(None)
#self.normalization.load(norm_file)
dataMatrix = np.array([d['target'] for d in data])
#dataMatrix = self.normalization.transform(dataMatrix)
for i in range(len(data)):
data[i]['target'] = dataMatrix[i]
self._data = data
def __init__(self, good_data, bad_data, balance=True, size=(1, 419, 419)):
super(LoaderTrain, self).__init__()
Plugin.setEnviron()
self._data = []
self._size = size
if (balance):
n_good = len(good_data)
n_bad = len(bad_data)
n_data = n_good if n_good < n_bad else n_bad
good_data = good_data[:n_data]
bad_data = bad_data[:n_data]
self._format_data(good_data, bad_data, self._data)
def _getAverageVol(self):
self._createFilenameTemplates()
Plugin.setEnviron()
listVol = self._getMrcVolumes()
avgVol = self._getFileName('avgMap')
npVol = npih.loadMrc(listVol[0], writable=False)
dType = npVol.dtype
npAvgVol = np.zeros(npVol.shape)
for vol in listVol:
npVol = npih.loadMrc(vol, writable=False)
npAvgVol += npVol
npAvgVol = np.divide(npAvgVol, len(listVol))
npih.saveMrc(npAvgVol.astype(dType), avgVol)
def testAlignVolumes(self):
Plugin.setEnviron()
volList = sorted(glob(self.volumes))
volRef = volList.pop(0)
maxScore = 0
npIh = NumpyImgHandler()
for vol in volList:
volRefNp = npIh.loadMrc(volRef)
volNp = npIh.loadMrc(vol)
volNpFp = np.fliplr(volNp)
axis, shifts, angles, score = npIh.alignVolumes(volNp, volRefNp)
axisf, shiftsf, anglesf, scoref = npIh.alignVolumes(
volNpFp, volRefNp)
print('scores : w/o flip- %03f w flip %03f' % (score, scoref))
if scoref > score:
print(
'angles:',
anglesf[0],
anglesf[1],
anglesf[2],
)
print(
'shifts:',
shiftsf[0],
shiftsf[1],
shiftsf[2],
)
npVol = npIh.applyTransforms(volNpFp, shiftsf, anglesf, axisf)
print('flipped map is better: ', vol)
else:
print(
'angles:',
angles[0],
angles[1],
angles[2],
)
print(
'shifts:',
shifts[0],
shifts[1],
shifts[2],
)
npVol = npIh.applyTransforms(volNp, shifts, angles, axis)
print('original map is better ', vol)
npIh.saveMrc(npVol, '/home/josuegbl/' + basename(vol))
def testAffinityProp(self):
from cryomethods.functions import MlMethods, NumpyImgHandler
Plugin.setEnviron()
volList = self._getVolList()
ml = MlMethods()
npIh = NumpyImgHandler()
dictNames = {}
groupDict = {}
matrix = npIh.getAllNpList(volList, 2)
# covMatrix, listNpVol = ml.getCovMatrixAuto(volList, 2)
# eigenVec, eigVal = ml.doPCA(covMatrix, 1)
# matrix = ml.getMatProjAuto(listNpVol, eigenVec)
labels = ml.doSklearnAffProp(matrix)
# labels = ml.doSklearnKmeans(matrix)
# labels = ml.doSklearnDBSCAN(matrix)
print(labels)
if labels is not None:
f = open('volumes_clustered.txt', 'w')
for vol, label in zip(volList, labels):
dictNames[vol] = label
destFn = '/home/josuegbl/PROCESSING/TESLA/projects/RNC_HTLnd2/MAPS' + basename(
vol)
copyFile(vol, destFn)
for key, value in sorted(dictNames.items()):
groupDict.setdefault(value, []).append(key)
counter = 0
for key, value in groupDict.items():
valueStr = ' '.join(value)
line = 'chimera %s\n' % valueStr
f.write(line)
counter += 1
avgFn = 'map_average_class_%02d.mrc' % counter
avgNp, _ = npIh.getAverageMap(value)
npIh.saveMrc(avgNp, avgFn)
f.close()
def _getMaps(self, coords):
Plugin.setEnviron()
coordMaps = self._getCoordMapFiles()
f = open(coordMaps)
for i, l in enumerate(f):
fn = self.protocol._getPath("volume_%02d.mrc" %i)
weigths = []
for coord in coords:
value = list(map(float, l.split()))
weigths.append(self._getDistanceWeigth(value, coord))
inputMaps = self.protocol._getMrcVolumes()
for j, (v, w) in enumerate(izip(inputMaps, weigths)):
npVol = NumpyImgHandler.loadMrc(v, False)
if j == 0:
dType = npVol.dtype
newMap = np.zeros(npVol.shape)
newMap += (w*npVol/sum(weigths))
NumpyImgHandler.saveMrc(newMap.astype(dType), fn)
f.close()
def _alignVolumes(self):
# Align all volumes inside a level
Plugin.setEnviron()
listVol = self._getPathMaps()
avgVol = self._getFileName('avgMap', lev=self._level)
npAvgVol = NumpyImgHandler.loadMrc(avgVol, writable=False)
dType = npAvgVol.dtype
for vol in listVol:
npVolAlign = NumpyImgHandler.loadMrc(vol, False)
npVolFlipAlign = np.fliplr(npVolAlign)
axis, shifts, angles, score = NumpyImgHandler.alignVolumes(npVolAlign, npAvgVol)
axisf, shiftsf, anglesf, scoref = NumpyImgHandler.alignVolumes(npVolFlipAlign,
npAvgVol)
if scoref > score:
npVol = NumpyImgHandler.applyTransforms(npVolFlipAlign, shiftsf, anglesf, axisf)
else:
npVol = NumpyImgHandler.applyTransforms(npVolAlign, shifts, angles, axis)
NumpyImgHandler.saveMrc(npVol.astype(dType), vol)
def _alignVolumes(self):
# Align all volumes
Plugin.setEnviron()
npIh = NumpyImgHandler()
listVol = self._getPathMaps('volume_????_filtered.mrc')
avgVol = self._getAvgMapFn()
npAvgVol = npIh.loadMrc(avgVol, writable=False)
dType = npAvgVol.dtype
for vol in listVol:
npVolAlign = self._getVolNp(vol)
npVolFlipAlign = np.fliplr(npVolAlign)
axis, shifts, angles, score = npIh.alignVolumes(npVolAlign, npAvgVol)
axisf, shiftsf, anglesf, scoref = npIh.alignVolumes(npVolFlipAlign,
npAvgVol)
if scoref > score:
npVol = npIh.applyTransforms(npVolFlipAlign, shiftsf, anglesf, axisf)
else:
npVol = npIh.applyTransforms(npVolAlign, shifts, angles, axis)
npIh.saveMrc(npVol.astype(dType), vol)
def analyzePCAStep(self):
self._createFilenameTemplates()
Plugin.setEnviron()
if self.alignment.get() == 0:
self.alignVols()
fnIn = self._getMrcVolumes()
self._getAverageVol()
avgVol = self._getFileName('avgMap')
npAvgVol = npih.loadMrc(avgVol, False)
dType = npAvgVol.dtype
iniVolNp = npih.loadMrc(fnIn[0], False)
dim = iniVolNp.shape[0]
lenght = dim**3
u, s, vh = np.linalg.svd(self._getCovMatrix())
vhDel = self._getvhDel(vh, s)
# -------------NEWBASE_AXIS-------------------------------------------
counter = 0
for eignRow in vhDel.T:
base = np.zeros(lenght)
for (vol, eigenCoef) in izip(fnIn, eignRow):
volInp = npih.loadMrc(vol, False)
volInpR = volInp.reshape(lenght)
volSubs = volInpR - npAvgVol.reshape(lenght)
base += volSubs * eigenCoef
volBase = base.reshape((dim, dim, dim))
nameVol = 'volume_base_%02d.mrc' % (counter)
print('-------------saving map %s-----------------' % nameVol)
npih.saveMrc(volBase.astype(dType), self._getExtraPath(nameVol))
counter += 1
matProj = []
baseMrc = self._getExtraPath("volume_base_??.mrc")
baseMrcFile = sorted(glob(baseMrc))
for vol in fnIn:
volNp = npih.loadMrc(vol, False)
restNpVol = volNp.reshape(lenght) - npAvgVol.reshape(lenght)
volRow = restNpVol.reshape(lenght)
rowCoef = []
for baseVol in baseMrcFile:
npVol = npih.loadMrc(baseVol, writable=False)
baseVol_row = npVol.reshape(lenght)
baseVol_col = baseVol_row.transpose()
projCoef = np.dot(volRow, baseVol_col)
rowCoef.append(projCoef)
matProj.append(rowCoef)
# obtaining volumes from coordinates-----------------------------------
os.makedirs(self._getExtraPath('reconstructed_vols'))
orignCount = 0
for projRow in matProj:
vol = np.zeros((dim, dim, dim))
for baseVol, proj in zip(baseMrcFile, projRow):
volNpo = npih.loadMrc(baseVol, False)
vol += volNpo * proj
finalVol = vol + npAvgVol
nameVol = 'volume_reconstructed_%02d.mrc' % (orignCount)
print('----------saving original_vols %s-------------' % nameVol)
npih.saveMrc(finalVol.astype(dType),
self._getExtraPath('reconstructed_vols', nameVol))
orignCount += 1
# difference b/w input vol and original vol-----------------------------
reconstMrc = self._getExtraPath("original_vols", "*.mrc")
reconstMrcFile = sorted(glob(reconstMrc))
diffCount = 0
os.makedirs(self._getExtraPath('volDiff'))
for a, b in zip(reconstMrcFile, fnIn):
volRec = npih.loadMrc(a, False)
volInpThree = npih.loadMrc(b, False)
volDiff = volRec - volInpThree
nameVol = 'volDiff_%02d.mrc' % (diffCount)
print('---------saving original_vols %s--------------' % nameVol)
npih.saveMrc(volDiff.astype(dType),
self._getExtraPath('volDiff', nameVol))
diffCount += 1
#save coordinates:
os.makedirs(self._getExtraPath('Coordinates'))
coorPath = self._getExtraPath('Coordinates')
mat_file = os.path.join(coorPath, 'matProj_splic')
np.save(mat_file, matProj)
def _pcaReconstruction(self, paramName=None):
Plugin.setEnviron()
if not os.path.exists(self.protocol._getExtraPath('Select_PC')):
os.mkdir(self.protocol._getExtraPath('Select_PC'))
nPCA = self.pcaCount.get()
print (nPCA)
avgVol = self.protocol._getPath('extramap_average.mrc')
npAvgVol = NumpyImgHandler.loadMrc(avgVol, False)
print ("average map is here")
dType = npAvgVol.dtype
fnIn = self.protocol._getMrcVolumes()
volNum = self.volNumb.get()
initVolNum = volNum - 1
iniVolNp = NumpyImgHandler.loadMrc(fnIn[0], False)
dim = iniVolNp.shape[0]
print (len(iniVolNp), "iniVolNp")
lenght = dim ** 3
reshapeVol = iniVolNp.reshape(lenght)
subsAvgVol= reshapeVol- npAvgVol.reshape(lenght)
# -------------------------covariance matrix----------------------
cov_matrix= np.load(
self.protocol._getExtraPath('CovMatrix', 'covMatrix.npy'))
print (len(cov_matrix), "cov_matrix")
u, s, vh = np.linalg.svd(cov_matrix)
sCut = int(self.pcaCount.get())
print (sCut, "scut")
vhDel = np.transpose(np.delete(vh, np.s_[sCut:vh.shape[1]], axis=0))
# --------------------obatining base-----------------------------
for eignRow in vhDel.T:
base = np.zeros(lenght)
# volSelect = self.protocol._getExtraPath('volume_id_%02d.mrc' % (self.volNumb.get()))
volSelect= fnIn[initVolNum:volNum]
print(volSelect, "volSelect")
for (vol, eigenCoef) in izip(volSelect,eignRow):
volInp = NumpyImgHandler.loadMrc(vol, False)
volInpR = volInp.reshape(lenght)
volSubs = volInpR - npAvgVol.reshape(lenght)
base += volSubs * eigenCoef
volBase = base.reshape((dim, dim, dim))
# break
break
nameVol = 'reconstruct_base_%02d.mrc' % (self.volNumb.get())
print('-------------saving map %s-----------------' % nameVol)
NumpyImgHandler.saveMrc(volBase.astype(dType),self.protocol._getExtraPath('Select_PC',nameVol))
#
# # ----------------matproj-----------------------------------------
matProj = []
baseMrc = self.protocol._getExtraPath('Select_PC', 'reconstruct_base_??.mrc')
baseMrcFile = sorted(glob(baseMrc))
volSelect = fnIn[initVolNum:volNum]
for vol in volSelect:
volNp = NumpyImgHandler.loadMrc(vol, False)
restNpVol = volNp.reshape(lenght) - npAvgVol.reshape(lenght)
volRow = restNpVol.reshape(lenght)
rowCoef = []
for baseVol in baseMrcFile:
npVol = NumpyImgHandler.loadMrc(baseVol, writable=False)
baseVol_row = npVol.reshape(lenght)
baseVol_col = baseVol_row.transpose()
projCoef = np.dot(volRow, baseVol_col)
rowCoef.append(projCoef)
matProj.append(rowCoef)
print (matProj, "matProj")
print ("length of bese file", len(baseMrcFile))
#
# # obtaining volumes from coordinates-----------------------------------
for projRow in matProj:
vol = np.zeros((dim, dim, dim))
for baseVol, proj in zip(baseMrcFile, projRow):
volNpo = NumpyImgHandler.loadMrc(baseVol, False)
vol += volNpo * proj
finalVol = vol + npAvgVol
nameRes = 'reconstruct_%02d.mrc' % (self.volNumb.get())
print('-------------saving reconstruct_vols %s-----------------' % nameRes)
NumpyImgHandler.saveMrc(finalVol.astype(dType),
self.protocol._getExtraPath('Select_PC', nameRes))
finalVol= fnIn[volNum]
orgVol = 'original_%02d.mrc' % (self.volNumb.get())
dst = self.protocol._getExtraPath('Select_PC', orgVol)
# NumpyImgHandler.saveMrc(finalVol.astype(dType),self.protocol._getExtraPath('Select_PC', orgVol))
copyfile(finalVol, dst)
def testPCA(self):
Plugin.setEnviron()
prot = Prot3DAutoClassifier(classMethod=1)
volList = sorted(glob(self.volumes))
matProj, _ = prot._doPCA(volList)
print(matProj)