def buildAKSmearedProtection_Feb9th(self):
df = dtedTools(dtedBase=self.dtedBasePath)
mc = mapCompare()
dted = mapTools()
max = mapTools()
for lati in range(63, 64):
for loni in range(-147, -146):
dtedFile = df.findDTEDFile(lat=lati, lon=loni)
if (dtedFile == []):
print('No dted cells at lat=' + str(lati) + 'lon=' + str(loni))
continue
dted.openMap(dtedFile)
max = mapTools()
max.createMaximumSmearedMap(model=dted, smear='card_diag')
max_fn = self.tmpPath + r'\t.tif'
max.outputMapFile(max_fn)
print(dtedFile)
Ak_fn = self.createAKTruthCell(lati, loni, dtedFile, overwrite=False, sampling='max')
if(Ak_fn == []):
print('Alaska DSM data was not created for some reason')
continue
output_fn = self.compareCellsBasePath + r'\Norm_' +str(lati) + 'w' + str(-1 * loni) + '.tif'
mc.createComparisonMap(compareMap_fn=max_fn, truth_fn=Ak_fn,
outputMap_fn=output_fn, view=True,
overwrite=False)
def createComparisonMap(self,
truth_fn,
compareMap_fn,
outputMap_fn,
overwrite=False,
view=False):
#create a difference map between the two
if (overwrite or not os.path.exists(outputMap_fn)):
truthMap = mapTools()
truthMap.openMap(truth_fn)
compareMap = mapTools()
compareMap.openMap(compareMap_fn)
if (not truthMap.compareMaps(compareMap)):
raise RuntimeError(
"Maps do not have equal extents, no map will be generated at "
+ outputMap_fn)
outMap = mapTools()
outMap.createEmptyMap(model=truthMap)
outMap.loadMap_Diff(truthMap=truthMap, compareMap=compareMap)
print("Output comparison map at " + outputMap_fn)
outMap.outputMapFile(outputMap_fn)
if (view):
QgsApplication.setPrefixPath("C:\\OSGeo4W64\\apps\\qgis", True)
qgs = QgsApplication([], True)
qgs.initQgis()
rw = RasterLayerWindow(rlayer1_path=truth_fn,
rlayer2_path=compareMap_fn,
comparison_path=outputMap_fn,
optionSet='prot')
rw.show()
qgs.exec_()
def feedback(self,
includeDir = False):
# provide some feedback
if not includeDir:
fileIn1 = self.inputMapFile.split('/')[-1]
if self.inputMapFile2 != '':
fileIn2 = self.inputMapFile2.split('/')[-1]
fileOut = self.outputMapFile.split('/')[-1]
else:
fileIn1 = self.inputMapFile
if self.inputMapFile2 != '':
fileIn2 = self.inputMapFile2
fileOut = self.outputMapFile
txt = 'MAPMASK Summary:\n'+\
'Input map file: {}\n'.format(fileIn1)
if self.inputMapFile2 != '':
txt += 'Input map 2 file: {}\n'.format(fileIn2)
txt += 'Output map file: {}'.format(fileOut)
self.runLog.writeToLog(txt)
Map = mapTools(mapName = self.outputMapFile,
logFile = self.runLog)
Map.printMapInfo()
def reportCroppedMapInfo(self):
# print the map info (grid size, num voxels)
self.runLog.writeToLog(
str='All generated maps should now be restricted to asym unit ' +
'with properties: ')
Map = mapTools(mapName=self.atomTaggedMap, logFile=self.runLog)
Map.printMapInfo()
def provideFeedback(self):
# provide some feedback
print '--------------------------'
print 'FFT Summary:'
print 'Input mtz file: {}'.format(self.inputMtzFile)
print 'Output map file: {}'.format(self.outputMapFile)
Map = mapTools(self.outputMapFile)
Map.printMapInfo()
print '--------------------------'
def mapConsistencyCheck(self): # this function determines whether the atom map and density # map calculated using SFALL and FFT are compatible - meaning # the grid dimensions/filtering are the same and the ordering # of the fast, medium, and slow axes are identical. fftMap = mapTools(self.croppedDensityMap) fftMap.readHeader() sfallMap = mapTools(self.atomTaggedMap) sfallMap.readHeader() err = 'Checking that atom map (SFALL) and density map (FFT) are compatible...' self.runLog.writeToLog(str = err) if (sfallMap.gridsamp1 != fftMap.gridsamp1 or sfallMap.gridsamp2 != fftMap.gridsamp2 or sfallMap.gridsamp3 != fftMap.gridsamp3): err = 'Incompatible grid sampling found...' self.runLog.writeToLog(str = err) return False if (sfallMap.fastaxis != fftMap.fastaxis or sfallMap.medaxis != fftMap.medaxis or sfallMap.slowaxis != fftMap.slowaxis): err = 'Incompatible fast,med,slow axes ordering found...' self.runLog.writeToLog(str = err) return False if (sfallMap.numCols != fftMap.numCols or sfallMap.numRows != fftMap.numRows or sfallMap.numSecs != fftMap.numSecs): err = 'Incompatible number of rows, columns and sections...' self.runLog.writeToLog(str = err) return False if sfallMap.getMapSize() != fftMap.getMapSize(): err = 'Incompatible map file sizes' self.runLog.writeToLog(str = err) return False self.runLog.writeToLog(str = '---> success!') return True
def feedback(self):
# provide some feedback
print '--------------------------'
print 'MAPMASK Summary:'
print 'Input map file: {}'.format(self.inputMapFile)
if self.inputMapFile2 != '':
print 'Input map 2 file: {}'.format(self.inputMapFile2)
print 'Output map file: {}'.format(self.outputMapFile)
Map = mapTools(self.outputMapFile)
Map.printMapInfo()
print '--------------------------'
def provideFeedback(self):
# provide some feedback
print('--------------------------')
print('END map Summary:')
print('Input pdb file: {}'.format(self.pdbFile))
print('Input mtz file: {}'.format(self.mtzFile))
print('Output map file: {}'.format(self.outputMapFile))
Map = mapTools(self.outputMapFile)
Map.printMapInfo()
print('--------------------------')
def buildArrays(self):
for lat in range(24, 80):
for lon in range(-180, 0):
#doDTEDArrayProc
fn = self.findFile(lat, lon, isDted=True)
if fn == '':
continue
full_fn = self.baseDiffDir + '\\' + fn
r, c = self.latlonToIndex(lat, lon)
self.fileExists[r, c] = True
mp = mapTools()
mp.openMap(full_fn)
num = float((mp.maskedArr > 60.95).sum())
sz = float(mp.maskedArr.count())
percen = num / sz * 100
self.dtednuisArr[r, c] = percen
num = float((mp.maskedArr < -60.95).sum())
percen = num / sz * 100
self.dtedprotArr[r, c] = percen
#doTDTArrayProc
fn = self.findFile(lat, lon, isDted=False)
if fn == '':
continue
full_fn = self.baseDiffDir + '\\' + fn
r, c = self.latlonToIndex(lat, lon)
mp = mapTools()
mp.openMap(full_fn)
num = float((mp.maskedArr > 60.95).sum())
sz = float(mp.maskedArr.count())
percen = num / sz * 100
self.tdtnuisArr[r, c] = percen
(mp.npArr < -60.95)
num = float((mp.maskedArr < -60.95).sum())
percen = num / sz * 100
self.tdtprotArr[r, c] = percen
def provideFeedback(self, includeDir=False):
# provide some feedback
if not includeDir:
fileIn = self.inputMtzFile.split("/")[-1]
fileOut = self.outputMapFile.split("/")[-1]
else:
fileIn = self.inputMtzFile
fileOut = self.outputMapFile
txt = "FFT Summary:\n" + "Input mtz file: {}\n".format(fileIn) + "Output map file: {}".format(fileOut)
self.runLog.writeToLog(txt)
Map = mapTools(mapName=self.outputMapFile, logFile=self.runLog)
Map.printMapInfo()
def provideFeedback(self,
includeDir=False):
# provide some feedback
if not includeDir:
fileIn = self.inputMtzFile.split('/')[-1]
fileOut = self.outputMapFile.split('/')[-1]
else:
fileIn = self.inputMtzFile
fileOut = self.outputMapFile
self.runLog.writeToLog('FFT Summary:\n' +
'Input mtz file: {}\n'.format(fileIn) +
'Output map file: {}'.format(fileOut))
Map = mapTools(mapName=self.outputMapFile, logFile=self.runLog)
Map.printMapInfo()
def provideFeedback(self,
includeDir=False):
# provide some feedback
if not includeDir:
fileIn = self.inputPDBfile.split('/')[-1]
fileOut = self.outputFile.split('/')[-1]
else:
fileIn = self.inputPDBfile
fileOut = self.outputFile
self.runLog.writeToLog(
'SFALL Summary:\nInput pdb file: {}\nOutput file: {}'.format(
fileIn, fileOut))
if self.task == 'atom map':
Map = mapTools(mapName=self.outputMapFile, logFile=self.runLog)
Map.printMapInfo()
def doGDALWarp(self,
modelCell_fn,
dataCells_fn,
output_fn,
resampletxt,
overwrite=False,
lon=None,
extent=None):
# Good explaination on https://www.programcreek.com/python/example/116446/gdal.Warp
# Also the classic python reference https://gdal.org/python/osgeo.gdal-module.html#Warp
# and the main C++ Reference gdal.org/programs/gdalwarp.html
# for the resampling modes the gdal constants https://gdal.org/python/osgeo.gdalconst-module.html
if (not overwrite and os.path.exists(output_fn)):
print('Cell at ' + output_fn +
' already exists and will NOT be overwritten')
return
mt = mapTools()
mt.openMap(modelCell_fn)
if (resampletxt == 'near'):
_resampleAlg = gdalconst.GRA_NearestNeighbour
elif (resampletxt == 'max'):
_resampleAlg = gdalconst.GRA_Max
else:
print('Value for resample text is out of bounds for this function')
return ()
if (
lon != None
): #supports using a dted cell as a model that doesn't exist for a particular longitude
mt.left = float(lon) - mt.resX / 2
mt.right = float(lon + 1) + mt.resX / 2
print("Processing Gdal warp " + resampletxt + " for cell n" +
str(math.ceil(mt.bttm)) + "w" + str(math.ceil(mt.left)))
gdal.Warp(output_fn,
dataCells_fn,
dstSRS=mt.prj,
xRes=mt.resX,
yRes=mt.resY,
outputBounds=[mt.left, mt.bttm, mt.right, mt.top],
resampleAlg=_resampleAlg)
def createCompositeMap(
self,
cells,
output_fn,
resampletxt='max',
thinPixels=10
): #thinPixels is resolution to divide the original by
mt = mapTools()
mt.openMap(cells[0])
if (resampletxt == 'near'):
_resampleAlg = gdalconst.GRA_NearestNeighbour
elif (resampletxt == 'max'):
_resampleAlg = gdalconst.GRA_Max
print('Processing composite map!')
gdal.Warp(output_fn,
cells,
dstSRS=mt.prj,
xRes=mt.resX * thinPixels,
yRes=mt.resY * thinPixels,
resampleAlg=_resampleAlg)
def provideFeedback(self,
includeDir = False):
# provide some feedback
if includeDir is False:
fileIn = self.inputPDBfile.split('/')[-1]
fileOut = self.outputMapFile.split('/')[-1]
else:
fileIn = self.inputPDBfile
fileOut = self.outputMapFile
txt = 'SFALL Summary:\n'+\
'Input pdb file: {}\n'.format(fileIn)+\
'Output map file: {}'.format(fileOut)
self.runLog.writeToLog(txt)
Map = mapTools(mapName = self.outputMapFile,
logFile = self.runLog)
Map.printMapInfo()
def getAtomTaggedMap(self):
# run SFALL job to generate atom tagged map
self.printStepNumber()
sfall = SFALLjob(inputPDBfile=self.reorderedPDBFile,
outputDir=self.outputDir, VDWR=self.sfall_VDWR,
symmetrygroup=self.spaceGroup, runLog=self.runLog,
outputMapFile=self.atomTaggedMap,
gridDimensions=self.sfallGRIDdims)
success = sfall.run()
sfallMap = mapTools(mapName=self.atomTaggedMap)
self.axes = [sfallMap.fastaxis, sfallMap.medaxis, sfallMap.slowaxis]
self.gridSamps = [sfallMap.gridsamp1, sfallMap.gridsamp2,
sfallMap.gridsamp3]
if not success:
error(text='Failure to successfully generate tagged atom map',
log=self.runLog)
def getAtomTaggedMap(self): # run SFALL job to generate atom tagged map self.printStepNumber() sfall = SFALLjob(inputPDBfile = self.reorderedPDBFile, outputDir = self.outputDir, VDWR = self.sfall_VDWR, symmetrygroup = self.spaceGroup, gridDimensions = self.sfall_GRID, runLog = self.runLog) success = sfall.run() sfallMap = mapTools(mapName = sfall.outputMapFile) self.axes = [sfallMap.fastaxis, sfallMap.medaxis, sfallMap.slowaxis] self.gridSamps = [sfallMap.gridsamp1, sfallMap.gridsamp2, sfallMap.gridsamp3] self.atomTaggedMap = sfall.outputMapFile return success
def openRasterLayers(self,
normDiff_path=None,
lowLevelDiff_path=None,
terrainElev_path=None,
overwrite=True):
gt = gisTools()
mt = mapTools()
if (normDiff_path != None):
base = os.path.splitext(ntpath.basename(normDiff_path))[0]
mt.openMap(normDiff_path)
n = mt.geoTrans[3]
w = mt.geoTrans[0]
s = n + mt.geoTrans[5] * mt.sizeY
e = w + mt.geoTrans[1] * mt.sizeX
nw = gt.reprojectPt(src_epsg='EPSG:4326',
dst_epsg=self.epsg,
tuple=[w, n, 0])
se = gt.reprojectPt(src_epsg='EPSG:4326',
dst_epsg=self.epsg,
tuple=[e, s, 0])
_xres = (se[0] - nw[0]) / mt.sizeX
_yres = (se[1] - nw[1]) / mt.sizeY
normTmp = self.tmpPath + r'\reproj-' + base + '.tif'
if (not os.path.exists(normTmp) or overwrite == True):
gdal.Warp(
normTmp,
normDiff_path,
dstSRS=self.epsg,
outputBounds=(nw[0], se[1], se[0], nw[1]),
xRes=_xres,
yRes=_yres
) # Got to reproject the files to view them with the background
# gdal.Warp(normTmp, normDiff_path, dstSRS=self.epsg) # Got to reproject the files to view them with the background
# mt.openMap(normTmp)
# m = mt.geoTrans
# s = mt.sizeY
# s1 = mt.sizeX
self.normLayer = QgsRasterLayer(normTmp, "Norm Diff")
self.normLayerHill = QgsRasterLayer(normTmp, "Norm Hill")
self.setRenderHillshade(layer=self.normLayerHill, zf=4)
self.normLayer.loadNamedStyle('NormMode.qml')
else:
self.normLayer = QgsRasterLayer(normDiff_path, "Norm Diff")
self.normLayerHill = QgsRasterLayer(normDiff_path, "Norm Diff")
if (lowLevelDiff_path != None):
base = os.path.splitext(ntpath.basename(lowLevelDiff_path))[0]
llvlTmp = self.tmpPath + r'\reproj-' + base + '.tif'
if (not os.path.exists(llvlTmp) or overwrite == True):
gdal.Warp(
llvlTmp, lowLevelDiff_path, dstSRS=self.epsg
) # Got to reproject the files to view them with the background
self.lowLevelLayer = QgsRasterLayer(llvlTmp, "Low Level Diff")
self.lowLevelLayerHill = QgsRasterLayer(llvlTmp, "Low Level Hill")
self.setRenderHillshade(layer=self.lowLevelLayerHill, zf=4)
self.lowLevelLayer.loadNamedStyle('NormMode.qml')
else:
self.lowLevelLayer = QgsRasterLayer(lowLevelDiff_path,
"LowLevl Diff")
self.lowLevelLayerHill = QgsRasterLayer(lowLevelDiff_path,
"LowLevl Diff")
if (terrainElev_path != None):
base = os.path.splitext(ntpath.basename(terrainElev_path))[0]
terrainTmp = self.tmpPath + r'\reproj-' + base + '.tif'
if (not os.path.exists(terrainTmp) or overwrite == True):
gdal.Warp(
terrainTmp, terrainElev_path, dstSRS=self.epsg
) # Got to reproject the files to view them with the background
self.terrainElevLayer = QgsRasterLayer(terrainTmp,
"Terrain Elevation")
self.setRenderHillshade(layer=self.terrainElevLayer, zf=4)
def runPipeline(self):
# read input file first
success = self.readInputFile()
if success is False:
return 1
# create log file
self.runLog = logFile('{}/{}_runLog_2.txt'.format(self.outputDir,self.jobName))
# run pdbcur job
pdbcur = PDBCURjob(self.pdbcurPDBinputFile,self.outputDir,self.runLog)
success = pdbcur.run()
if success is False:
return 2
self.PDBCURoutputFile = pdbcur.outputPDBfile
# reorder atoms in PDB file
self.renumberPDBFile()
# get space group from PDB file
success = self.getSpaceGroup()
if success is False:
return 3
# run SFALL job
sfall = SFALLjob(self.reorderedPDBFile,self.outputDir,self.sfall_VDWR,
self.spaceGroup,self.sfall_GRID,'ATMMOD',self.runLog)
success = sfall.run()
if success is False:
return 4
# run FFT job
sfallMap = mapTools(sfall.outputMapFile)
axes = [sfallMap.fastaxis,sfallMap.medaxis,sfallMap.slowaxis]
gridSamps = [sfallMap.gridsamp1,sfallMap.gridsamp2,sfallMap.gridsamp3]
labelsInit = ['FP_'+self.initPDB,'SIGFP_'+self.initPDB,'FOM_'+self.initPDB,'PHIC_'+self.initPDB]
labelsLater = ['FP_'+self.laterPDB,'SIGFP_'+self.laterPDB,'FOM_'+self.laterPDB,'PHIC_'+self.laterPDB]
if self.densMapType != 'END':
fft = FFTjob(self.densMapType,self.FOMweight,self.reorderedPDBFile,self.inputMtzFile,
self.outputDir,axes,gridSamps,labelsLater,labelsInit,self.runLog)
success = fft.run()
else:
# run END job if required (may take time to run!!)
endInputPDB = self.pdbcurPDBinputFile
endInputMTZ = ''.join(endInputPDB.split('.')[:-1]+['.mtz'])
endInputEFF = ''.join(endInputPDB.split('.')[:-1]+['.eff'])
end = ENDjob(endInputPDB,endInputMTZ,endInputEFF,self.outputDir,gridSamps,self.runLog)
success = end.run()
if success is False:
return 5
# crop fft and atom-tagged maps to asymmetric unit:
mapmask1 = MAPMASKjob(sfall.outputMapFile,'',self.outputDir,self.runLog)
success = mapmask1.crop2AsymUnit()
if success is False:
return 6
# choose correct density map to include in MAPMASK cropping below
if self.densMapType != 'END':
inputDensMap = fft.outputMapFile
else:
inputDensMap = end.outputMapFile
# switch END map axes to match SFALL atom-tagged map if required
if self.densMapType == 'END':
mapmask_ENDmap = MAPMASKjob(inputDensMap,'',self.outputDir,self.runLog)
success = mapmask_ENDmap.switchAxisOrder(axes,self.spaceGroup)
if success is False:
return 7.0
else: inputDensMap = mapmask_ENDmap.outputMapFile
# run MAPMASK job to crop fft density map to asym unit
mapmask2 = MAPMASKjob(inputDensMap,'',self.outputDir,self.runLog)
success = mapmask2.crop2AsymUnit()
if success is False:
return 7.1
# run MAPMASK job to crop fft density map to same grid
# sampling dimensions as SFALL atom map
mapmask3 = MAPMASKjob(mapmask2.outputMapFile,mapmask1.outputMapFile,self.outputDir,self.runLog)
success = mapmask3.cropMap2Map()
if success is False:
return 8
# perform map consistency check between cropped fft and sfall maps
fftMap = mapTools(mapmask3.outputMapFile)
fftMap.readHeader()
sfallMap = mapTools(mapmask1.outputMapFile)
sfallMap.readHeader()
success = self.mapConsistencyCheck(sfallMap,fftMap)
if success is False:
return 9
else:
return 0
from gdalRasterTools import gdalRasterTool
from mapComparisonTools import mapCompare
from RasterHandler import RasterHandler
if __name__ == "__main__":
dtedBasePath = r'C:\0-Data\AK_Elev\dted1\dted'
outputBasePath = r'C:\0-Data\0-GeneratedData\tmp'
lati = 60
loni = -150
df = dtedTools(dtedBase=dtedBasePath)
dtedFile = df.findDTEDFile(lat=lati, lon=loni)
if (dtedFile == []):
print('No dted cells at lat=' + str(lati) + 'lon=' + str(loni))
exit()
print(dtedFile)
dted = mapTools()
dted.openMap(dtedFile)
max = mapTools()
max.createMaximumSmearedMap(model=dted, smear='card_diag')
# max.createEmptyMap(model=dted)
# max.npArr = np.copy(dted.npArr)
# if(smear == 'cardinal' or smear == 'card_diag'):
# t = mapTools()
# t.createShiftedMap(model=dted, shift='n', fillValue=100)
# max.npArr = np.maximum(max.npArr, t.npArr)
# t.createShiftedMap(model=dted, shift='s', fillValue=100)
# max.npArr = np.maximum(max.npArr, t.npArr)
# t.createShiftedMap(model=dted, shift='e', fillValue=100)
# max.npArr = np.maximum(max.npArr, t.npArr)
# t.createShiftedMap(model=dted, shift='w', fillValue=100)