def _saveAverageVol(self):
listVol = self._getPathMaps('volume_????_filtered.mrc')
avgVol = self._getAvgMapFn()
npAvgVol, dType = self._doAverageMap(listVol)
print("Dtype: ", dType)
if dType == 'float64':
dType = 'float32'
NumpyImgHandler.saveMrc(npAvgVol.astype(dType), avgVol)
def _getAverageVol(self, listVol=[]):
listVol = self._getPathMaps() if not bool(listVol) else listVol
try:
avgVol = self._getFileName('avgMap', lev=self._level)
except:
avgVol = self._getPath('map_average.mrc')
npIh = NumpyImgHandler()
npAvgVol, _ = npIh.getAverageMap(listVol)
npIh.saveMrc(npAvgVol, avgVol)
def _reconstructMap(self, matProj):
from glob import glob
listBaseVol = glob('volume_base*.mrc')
sortedList = sorted(listBaseVol)
listNpBase, dType = self._mrcToNp(sortedList)
volNpList = np.dot(matProj, listNpBase)
dim = int(round(volNpList.shape[1]**(1. / 3)))
npIh = NumpyImgHandler()
for i, npVol in enumerate(volNpList):
npVolT = np.transpose(npVol)
volNp = npVolT.reshape((dim, dim, dim))
nameVol = 'volume_reconstructed_%02d.mrc' % (i + 1)
npIh.saveMrc(volNp.astype(dType), nameVol)
def _getAverageVol(self):
self._createFilenameTemplates()
Plugin.setEnviron()
listVol = self._getMrcVolumes()
avgVol = self._getFileName('avgMap')
npVol = npih.loadMrc(listVol[0], writable=False)
dType = npVol.dtype
npAvgVol = np.zeros(npVol.shape)
for vol in listVol:
npVol = npih.loadMrc(vol, writable=False)
npAvgVol += npVol
npAvgVol = np.divide(npAvgVol, len(listVol))
npih.saveMrc(npAvgVol.astype(dType), avgVol)
def testAlignVolumes(self):
Plugin.setEnviron()
volList = sorted(glob(self.volumes))
volRef = volList.pop(0)
maxScore = 0
npIh = NumpyImgHandler()
for vol in volList:
volRefNp = npIh.loadMrc(volRef)
volNp = npIh.loadMrc(vol)
volNpFp = np.fliplr(volNp)
axis, shifts, angles, score = npIh.alignVolumes(volNp, volRefNp)
axisf, shiftsf, anglesf, scoref = npIh.alignVolumes(
volNpFp, volRefNp)
print('scores : w/o flip- %03f w flip %03f' % (score, scoref))
if scoref > score:
print(
'angles:',
anglesf[0],
anglesf[1],
anglesf[2],
)
print(
'shifts:',
shiftsf[0],
shiftsf[1],
shiftsf[2],
)
npVol = npIh.applyTransforms(volNpFp, shiftsf, anglesf, axisf)
print('flipped map is better: ', vol)
else:
print(
'angles:',
angles[0],
angles[1],
angles[2],
)
print(
'shifts:',
shifts[0],
shifts[1],
shifts[2],
)
npVol = npIh.applyTransforms(volNp, shifts, angles, axis)
print('original map is better ', vol)
npIh.saveMrc(npVol, '/home/josuegbl/' + basename(vol))
def _getPathMaps(self):
inputObj = self.inputVolumes.get()
filesPath = []
for i in inputObj:
a = npih.getImageLocation(i)
filesPath.append(a)
return sorted(glob(filesPath))
def __getitem__(self, item):
data = self._data[item]
img_path = data['file']
img = NumpyImgHandler.loadMrc(img_path)
img = normalize(img)
img = np.resize(img, self._size)
img = torch.from_numpy(img)
return {'image': img, 'path': img_path}
def open_image(self, filename):
#img = NumpyImgHandler.loadMrc(filename)
img = NumpyImgHandler.load(filename)
_min = img.min()
_max = img.max()
img = (img - _min) / (_max - _min)
img = np.resize(img, (1, 512, 512))
return torch.from_numpy(img)
def _estimatePCA(self):
from cryomethods.functions import MlMethods
ml = MlMethods()
listVol = self._getFinalMaps()
volNp = NumpyImgHandler.loadMrc(listVol[0], False)
dim = volNp.shape[0]
dType = volNp.dtype
matProj, newBaseAxis = ml.doPCAuto(listVol, 2, 1)
for i, volNewBaseList in enumerate(newBaseAxis):
volBase = volNewBaseList.reshape((dim, dim, dim))
nameVol = 'volume_base_%02d.mrc' % (i+1)
NumpyImgHandler.saveMrc(volBase.astype(dType),
self._getLevelPath(self._level, nameVol))
return matProj
def open_image(self, filename):
img = NumpyImgHandler.loadMrc(filename)
# _min = img.min()
# _max = img.max()
# img = (img - _min) / (_max - _min)
psd = calcAvgPsd(img[0, :, :], windows_size=512, step_size=128)
# img = np.resize(img, (1, 512, 512))
return torch.from_numpy(np.float32(psd))
def testCorrectAnisotrophy(self):
Plugin.setEnviron()
npIh = NumpyImgHandler()
volNp = npIh.loadMrc(self.volume1)
vol, volFt = correctAnisotropy(volNp, 0.5, 0.55, 0.1, 0.45)
print("shape: ", vol.shape)
npIh.saveMrc(vol.astype(volNp.dtype), "corrected.mrc")
npIh.saveMrc(volFt.astype(volNp.dtype), "corrected_ft.mrc")
def computeOccupancyStep(self):
volume_path = self.vol.get().getFileName()
vol = NumpyImgHandler.loadMrc(volume_path)
mask_path = self.mask_in_molecule.get().getFileName()
mask = NumpyImgHandler.loadMrc(mask_path)
voxel_size = self.vol.get().getSamplingRate()
min_res = self.min_res.get()
max_res = self.max_res.get()
num_points = 10 # NOTE: hardcoded
protein_threshold = self.protein_threshold.get()
f_voxel_width = self.f_voxel_width.get()
omap = occupancy(vol, mask, voxel_size, min_res, max_res, num_points,
protein_threshold, f_voxel_width)
omap_path = self._getExtraPath('omap.mrc')
NumpyImgHandler.saveMrc(omap, omap_path)
def mergeClassesStep(self):
if self.doGrouping:
from cryomethods.functions import NumpyImgHandler
npIh = NumpyImgHandler()
makePath(self._getLevelPath(self._level))
listVol = self._getFinalMaps()
matrix = npIh.getAllNpList(listVol, 2)
labels = self._clusteringData(matrix)
clsChange = 0
prevStar = self._getFileName('rawFinalData')
pTable = Table()
origStar = self._getFileName('input_star', lev=1, rLev=1)
opticsTable = Table(fileName=origStar, tableName='optics')
print("OPTABLE: ", origStar, opticsTable.size())
for row in pTable.iterRows(prevStar, key="rlnClassNumber",
tableName='particles'):
clsPart = row.rlnClassNumber
newClass = labels[clsPart - 1] + 1
newRow = row._replace(rlnClassNumber=newClass)
if not newClass == clsChange:
if not clsChange == 0:
self.writeStar(fn, ouTable, opticsTable)
clsChange = newClass
fn = self._getFileName('input_star', lev=self._level,
rLev=newClass)
tableIn = Table(fileName=prevStar, tableName='particles')
cols = [str(c) for c in tableIn.getColumnNames()]
ouTable = Table(columns=cols, tableName='particles')
ouTable.addRow(*newRow)
print("mergeClassesStep ouTable.size: ", ouTable.size())
self.writeStar(fn, ouTable, opticsTable)
else:
prevData = self._getFileName('rawFinalData')
finalData = self._getFileName('finalData')
prevModel = self._getFileName('rawFinalModel')
finalModel = self._getFileName('finalModel')
copyFile(prevData, finalData)
copyFile(prevModel, finalModel)
def predict(model, device, data_loader, trainset):
"""
Method to predict using the neuronal network
"""
model.eval()
results = []
psd_list = []
with torch.no_grad():
for data in data_loader:
# Move tensors to the configured device
filename = 'psd/' + os.path.basename(data['name'][0]) + '_psd.mrc'
image = data['image']
NumpyImgHandler.saveMrc(np.float32(image.numpy()), filename)
image = image.to(device)
# Forward pass
output = model(image)
output = trainset.normalization.inv_transform(output.cpu().numpy())
# Save results
results.append(output[0])
psd_list.append(filename)
return psd_list, results
def _getMaps(self, coords):
Plugin.setEnviron()
coordMaps = self._getCoordMapFiles()
f = open(coordMaps)
for i, l in enumerate(f):
fn = self.protocol._getPath("volume_%02d.mrc" %i)
weigths = []
for coord in coords:
value = list(map(float, l.split()))
weigths.append(self._getDistanceWeigth(value, coord))
inputMaps = self.protocol._getMrcVolumes()
for j, (v, w) in enumerate(izip(inputMaps, weigths)):
npVol = NumpyImgHandler.loadMrc(v, False)
if j == 0:
dType = npVol.dtype
newMap = np.zeros(npVol.shape)
newMap += (w*npVol/sum(weigths))
NumpyImgHandler.saveMrc(newMap.astype(dType), fn)
f.close()
def computeBFactorStep(self):
volume_path = self.vol.get().getFileName()
vol = NumpyImgHandler.loadMrc(volume_path)
mask_path = self.mask_in_molecule.get().getFileName()
mask = NumpyImgHandler.loadMrc(mask_path)
voxel_size = self.vol.get().getSamplingRate()
min_res = self.min_res.get()
max_res = self.max_res.get()
num_points = 10 # NOTE: hardcoded
noise_threshold = self.noise_threshold.get()
f_voxel_width = self.f_voxel_width.get()
only_above_noise = self.only_above_noise.get()
bmap = bfactor(vol, mask, voxel_size, min_res, max_res,
num_points, noise_threshold, f_voxel_width,
only_above_noise)
bmap_path = self._getExtraPath('bmap.mrc')
NumpyImgHandler.saveMrc(bmap, bmap_path)
def _getCovMatrix(self):
if self.alignment.get() == 0:
self.alignVols()
fnIn = self._getMrcVolumes()
else:
fnIn = self._getMrcVolumes()
self._getAverageVol()
avgVol = self._getFileName('avgMap')
npAvgVol = npih.loadMrc(avgVol, False)
dType = npAvgVol.dtype
iniVolNp = npih.loadMrc(fnIn[0], False)
dim = iniVolNp.shape[0]
lenght = dim**3
cov_matrix = []
for vol in fnIn:
volNp = npih.loadMrc(vol, False)
volList = volNp.reshape(lenght)
row = []
# Now, using diff volume to estimate PCA
b = volList - npAvgVol.reshape(lenght)
for j in fnIn:
npVol = npih.loadMrc(j, writable=False)
volList_a = npVol.reshape(lenght)
volList_two = volList_a - npAvgVol.reshape(lenght)
temp_a = np.corrcoef(volList_two, b).item(1)
row.append(temp_a)
cov_matrix.append(row)
os.makedirs(self._getExtraPath('CovMatrix'))
covPath = self._getExtraPath('CovMatrix')
CovMatrix = os.path.join(covPath, 'covMatrix')
np.save(CovMatrix, cov_matrix)
CovMatData = np.load(self._getExtraPath('CovMatrix', 'covMatrix.npy'))
return CovMatData
def _alignVolumes(self):
# Align all volumes
Plugin.setEnviron()
npIh = NumpyImgHandler()
listVol = self._getPathMaps('volume_????_filtered.mrc')
avgVol = self._getAvgMapFn()
npAvgVol = npIh.loadMrc(avgVol, writable=False)
dType = npAvgVol.dtype
for vol in listVol:
npVolAlign = self._getVolNp(vol)
npVolFlipAlign = np.fliplr(npVolAlign)
axis, shifts, angles, score = npIh.alignVolumes(npVolAlign, npAvgVol)
axisf, shiftsf, anglesf, scoref = npIh.alignVolumes(npVolFlipAlign,
npAvgVol)
if scoref > score:
npVol = npIh.applyTransforms(npVolFlipAlign, shiftsf, anglesf, axisf)
else:
npVol = npIh.applyTransforms(npVolAlign, shifts, angles, axis)
npIh.saveMrc(npVol.astype(dType), vol)
def alignVols(self):
self._getAverageVol()
avgVol = self._getFileName('avgMap')
npAvgVol = npih.loadMrc(avgVol, False)
dType = npAvgVol.dtype
fnIn = self._getMrcVolumes()
for vols in fnIn:
npVolAlign = npih.loadMrc(vols, False)
npVolFlipAlign = np.fliplr(npVolAlign)
axis, shifts, angles, score = npih.alignVolumes(
npVolAlign, npAvgVol)
axisf, shiftsf, anglesf, scoref = npih.alignVolumes(
npVolFlipAlign, npAvgVol)
if scoref > score:
npVol = npih.applyTransforms(npVolFlipAlign, shiftsf, anglesf,
axisf)
else:
npVol = npih.applyTransforms(npVolAlign, shifts, angles, axis)
npih.saveMrc(npVol.astype(dType), vols)
def _alignVolumes(self):
# Align all volumes inside a level
Plugin.setEnviron()
listVol = self._getPathMaps()
avgVol = self._getFileName('avgMap', lev=self._level)
npAvgVol = NumpyImgHandler.loadMrc(avgVol, writable=False)
dType = npAvgVol.dtype
for vol in listVol:
npVolAlign = NumpyImgHandler.loadMrc(vol, False)
npVolFlipAlign = np.fliplr(npVolAlign)
axis, shifts, angles, score = NumpyImgHandler.alignVolumes(npVolAlign, npAvgVol)
axisf, shiftsf, anglesf, scoref = NumpyImgHandler.alignVolumes(npVolFlipAlign,
npAvgVol)
if scoref > score:
npVol = NumpyImgHandler.applyTransforms(npVolFlipAlign, shiftsf, anglesf, axisf)
else:
npVol = NumpyImgHandler.applyTransforms(npVolAlign, shifts, angles, axis)
NumpyImgHandler.saveMrc(npVol.astype(dType), vol)
def testAffinityProp(self):
from cryomethods.functions import MlMethods, NumpyImgHandler
Plugin.setEnviron()
volList = self._getVolList()
ml = MlMethods()
npIh = NumpyImgHandler()
dictNames = {}
groupDict = {}
matrix = npIh.getAllNpList(volList, 2)
# covMatrix, listNpVol = ml.getCovMatrixAuto(volList, 2)
# eigenVec, eigVal = ml.doPCA(covMatrix, 1)
# matrix = ml.getMatProjAuto(listNpVol, eigenVec)
labels = ml.doSklearnAffProp(matrix)
# labels = ml.doSklearnKmeans(matrix)
# labels = ml.doSklearnDBSCAN(matrix)
print(labels)
if labels is not None:
f = open('volumes_clustered.txt', 'w')
for vol, label in zip(volList, labels):
dictNames[vol] = label
destFn = '/home/josuegbl/PROCESSING/TESLA/projects/RNC_HTLnd2/MAPS' + basename(
vol)
copyFile(vol, destFn)
for key, value in sorted(dictNames.items()):
groupDict.setdefault(value, []).append(key)
counter = 0
for key, value in groupDict.items():
valueStr = ' '.join(value)
line = 'chimera %s\n' % valueStr
f.write(line)
counter += 1
avgFn = 'map_average_class_%02d.mrc' % counter
avgNp, _ = npIh.getAverageMap(value)
npIh.saveMrc(avgNp, avgFn)
f.close()
def _getVolNp(self, vol):
mapNp = NumpyImgHandler.loadMrc(vol, False)
std = 2 * mapNp.std()
npMask = 1 * (mapNp >= std)
mapNp = mapNp * npMask
return mapNp, npMask
def analyzePCAStep(self):
self._createFilenameTemplates()
Plugin.setEnviron()
if self.alignment.get() == 0:
self.alignVols()
fnIn = self._getMrcVolumes()
self._getAverageVol()
avgVol = self._getFileName('avgMap')
npAvgVol = npih.loadMrc(avgVol, False)
dType = npAvgVol.dtype
iniVolNp = npih.loadMrc(fnIn[0], False)
dim = iniVolNp.shape[0]
lenght = dim**3
u, s, vh = np.linalg.svd(self._getCovMatrix())
vhDel = self._getvhDel(vh, s)
# -------------NEWBASE_AXIS-------------------------------------------
counter = 0
for eignRow in vhDel.T:
base = np.zeros(lenght)
for (vol, eigenCoef) in izip(fnIn, eignRow):
volInp = npih.loadMrc(vol, False)
volInpR = volInp.reshape(lenght)
volSubs = volInpR - npAvgVol.reshape(lenght)
base += volSubs * eigenCoef
volBase = base.reshape((dim, dim, dim))
nameVol = 'volume_base_%02d.mrc' % (counter)
print('-------------saving map %s-----------------' % nameVol)
npih.saveMrc(volBase.astype(dType), self._getExtraPath(nameVol))
counter += 1
matProj = []
baseMrc = self._getExtraPath("volume_base_??.mrc")
baseMrcFile = sorted(glob(baseMrc))
for vol in fnIn:
volNp = npih.loadMrc(vol, False)
restNpVol = volNp.reshape(lenght) - npAvgVol.reshape(lenght)
volRow = restNpVol.reshape(lenght)
rowCoef = []
for baseVol in baseMrcFile:
npVol = npih.loadMrc(baseVol, writable=False)
baseVol_row = npVol.reshape(lenght)
baseVol_col = baseVol_row.transpose()
projCoef = np.dot(volRow, baseVol_col)
rowCoef.append(projCoef)
matProj.append(rowCoef)
# obtaining volumes from coordinates-----------------------------------
os.makedirs(self._getExtraPath('reconstructed_vols'))
orignCount = 0
for projRow in matProj:
vol = np.zeros((dim, dim, dim))
for baseVol, proj in zip(baseMrcFile, projRow):
volNpo = npih.loadMrc(baseVol, False)
vol += volNpo * proj
finalVol = vol + npAvgVol
nameVol = 'volume_reconstructed_%02d.mrc' % (orignCount)
print('----------saving original_vols %s-------------' % nameVol)
npih.saveMrc(finalVol.astype(dType),
self._getExtraPath('reconstructed_vols', nameVol))
orignCount += 1
# difference b/w input vol and original vol-----------------------------
reconstMrc = self._getExtraPath("original_vols", "*.mrc")
reconstMrcFile = sorted(glob(reconstMrc))
diffCount = 0
os.makedirs(self._getExtraPath('volDiff'))
for a, b in zip(reconstMrcFile, fnIn):
volRec = npih.loadMrc(a, False)
volInpThree = npih.loadMrc(b, False)
volDiff = volRec - volInpThree
nameVol = 'volDiff_%02d.mrc' % (diffCount)
print('---------saving original_vols %s--------------' % nameVol)
npih.saveMrc(volDiff.astype(dType),
self._getExtraPath('volDiff', nameVol))
diffCount += 1
#save coordinates:
os.makedirs(self._getExtraPath('Coordinates'))
coorPath = self._getExtraPath('Coordinates')
mat_file = os.path.join(coorPath, 'matProj_splic')
np.save(mat_file, matProj)
def _pcaReconstruction(self, paramName=None):
Plugin.setEnviron()
if not os.path.exists(self.protocol._getExtraPath('Select_PC')):
os.mkdir(self.protocol._getExtraPath('Select_PC'))
nPCA = self.pcaCount.get()
print (nPCA)
avgVol = self.protocol._getPath('extramap_average.mrc')
npAvgVol = NumpyImgHandler.loadMrc(avgVol, False)
print ("average map is here")
dType = npAvgVol.dtype
fnIn = self.protocol._getMrcVolumes()
volNum = self.volNumb.get()
initVolNum = volNum - 1
iniVolNp = NumpyImgHandler.loadMrc(fnIn[0], False)
dim = iniVolNp.shape[0]
print (len(iniVolNp), "iniVolNp")
lenght = dim ** 3
reshapeVol = iniVolNp.reshape(lenght)
subsAvgVol= reshapeVol- npAvgVol.reshape(lenght)
# -------------------------covariance matrix----------------------
cov_matrix= np.load(
self.protocol._getExtraPath('CovMatrix', 'covMatrix.npy'))
print (len(cov_matrix), "cov_matrix")
u, s, vh = np.linalg.svd(cov_matrix)
sCut = int(self.pcaCount.get())
print (sCut, "scut")
vhDel = np.transpose(np.delete(vh, np.s_[sCut:vh.shape[1]], axis=0))
# --------------------obatining base-----------------------------
for eignRow in vhDel.T:
base = np.zeros(lenght)
# volSelect = self.protocol._getExtraPath('volume_id_%02d.mrc' % (self.volNumb.get()))
volSelect= fnIn[initVolNum:volNum]
print(volSelect, "volSelect")
for (vol, eigenCoef) in izip(volSelect,eignRow):
volInp = NumpyImgHandler.loadMrc(vol, False)
volInpR = volInp.reshape(lenght)
volSubs = volInpR - npAvgVol.reshape(lenght)
base += volSubs * eigenCoef
volBase = base.reshape((dim, dim, dim))
# break
break
nameVol = 'reconstruct_base_%02d.mrc' % (self.volNumb.get())
print('-------------saving map %s-----------------' % nameVol)
NumpyImgHandler.saveMrc(volBase.astype(dType),self.protocol._getExtraPath('Select_PC',nameVol))
#
# # ----------------matproj-----------------------------------------
matProj = []
baseMrc = self.protocol._getExtraPath('Select_PC', 'reconstruct_base_??.mrc')
baseMrcFile = sorted(glob(baseMrc))
volSelect = fnIn[initVolNum:volNum]
for vol in volSelect:
volNp = NumpyImgHandler.loadMrc(vol, False)
restNpVol = volNp.reshape(lenght) - npAvgVol.reshape(lenght)
volRow = restNpVol.reshape(lenght)
rowCoef = []
for baseVol in baseMrcFile:
npVol = NumpyImgHandler.loadMrc(baseVol, writable=False)
baseVol_row = npVol.reshape(lenght)
baseVol_col = baseVol_row.transpose()
projCoef = np.dot(volRow, baseVol_col)
rowCoef.append(projCoef)
matProj.append(rowCoef)
print (matProj, "matProj")
print ("length of bese file", len(baseMrcFile))
#
# # obtaining volumes from coordinates-----------------------------------
for projRow in matProj:
vol = np.zeros((dim, dim, dim))
for baseVol, proj in zip(baseMrcFile, projRow):
volNpo = NumpyImgHandler.loadMrc(baseVol, False)
vol += volNpo * proj
finalVol = vol + npAvgVol
nameRes = 'reconstruct_%02d.mrc' % (self.volNumb.get())
print('-------------saving reconstruct_vols %s-----------------' % nameRes)
NumpyImgHandler.saveMrc(finalVol.astype(dType),
self.protocol._getExtraPath('Select_PC', nameRes))
finalVol= fnIn[volNum]
orgVol = 'original_%02d.mrc' % (self.volNumb.get())
dst = self.protocol._getExtraPath('Select_PC', orgVol)
# NumpyImgHandler.saveMrc(finalVol.astype(dType),self.protocol._getExtraPath('Select_PC', orgVol))
copyfile(finalVol, dst)