def buildCmds(self, inputPath, outputFile, headerSearchStr, searchDepth, sensor):
"""
"""
inputPath = os.path.abspath(inputPath)
outputFile = os.path.abspath(outputFile)
headersFilesList = []
if headerSearchStr.count('*') == 0:
raise Exception("The search string you have provided does not have any '*' - which is needed for searching.")
headersFilesList = self.getListOfFiles(inputPath, headerSearchStr, searchDepth)
sensorFact = ARCSISensorFactory()
sensorClass = sensorFact.getSensorClassFromName(sensor, False, None)
hdfFileLUT = dict()
for hdrFile in headersFilesList:
print(hdrFile)
sensorClass.extractHeaderParameters(hdrFile, None)
if sensor == 'sen2':
scnStr = sensorClass.spacecraftName.replace('-', '') + '_' + sensorClass.acquisitionTime.strftime("%Y%m%d")
else:
scnStr = sensor + '_' + sensorClass.acquisitionTime.strftime("%Y%m%d")
if scnStr not in hdfFileLUT:
hdfFileLUT[scnStr] = []
hdfFileLUT[scnStr].append(hdrFile)
arcsiUtils = ARCSIUtils()
for scnStr in hdfFileLUT:
arcsiUtils.writeList2File(hdfFileLUT[scnStr], outputFile+scnStr+".txt")
print("Completed.")
def checkFileVersions(self, inputFile, outputFile):
"""
"""
arcsiUtils = ARCSIUtils()
hdrFilesLst = arcsiUtils.readTextFile2List(inputFile)
granuleLUT = dict()
for hdrFile in hdrFilesLst:
print(hdrFile)
sensorFact = ARCSISensorFactory()
sensorClass = sensorFact.getSensorClassFromName(
'sen2', False, None)
sensorClass.extractHeaderParameters(hdrFile, None)
tileGranuleID = sensorClass.uniqueTileID
processVer = sensorClass.processingBaseline
genTime = sensorClass.generationTime
orbNum = sensorClass.orbitNumber
projName = sensorClass.projNameStr
scnTileID = tileGranuleID + '_' + orbNum + '_' + projName
print(scnTileID)
if scnTileID not in granuleLUT:
granuleLUT[scnTileID] = list()
granuleLUT[scnTileID].append(
Sen2InfoObj(hdrFile, tileGranuleID, processVer, genTime,
orbNum, projName))
sensorClass = None
outHdrs = list()
for scnTileID in granuleLUT:
if len(granuleLUT[scnTileID]) > 1:
maxGenTime = 0
maxHdrHeader = ""
first = True
for sen2Info in granuleLUT[scnTileID]:
cGenTime = sen2Info.generationTime
if first:
first = False
maxHdrHeader = sen2Info.hdrFile
maxGenTime = cGenTime
elif cGenTime > maxGenTime:
maxHdrHeader = sen2Info.hdrFile
maxGenTime = cGenTime
outHdrs.append(maxHdrHeader)
else:
outHdrs.append(granuleLUT[scnTileID][0].hdrFile)
arcsiUtils.writeList2File(outHdrs, outputFile)
def checkExtractedFiles(self, inputDIR, outputFile, headerEnding,
sensorStr, archivesDIR):
inputDIR = os.path.abspath(inputDIR)
archivesDIR = os.path.abspath(archivesDIR)
archList = self.getListOfArchives(archivesDIR)
dirList = glob.glob(os.path.join(inputDIR, "*"))
sensorFact = ARCSISensorFactory()
outFileList = open(outputFile, 'w')
foundErr = False
for dir in dirList:
foundErr = False
if os.path.isdir(dir):
print(dir)
hdrList = self.getListOfFiles(dir, headerEnding)
if len(hdrList) == 0:
print("Error: No header present")
foundErr = True
elif len(hdrList) > 1:
print(
"Error: Multiple header files present - don't know what to do with that!"
)
foundErr = True
else:
sensorClass = sensorFact.getSensorClassFromName(
sensorStr, False, None)
sensorClass.extractHeaderParameters(hdrList[0], "")
if not sensorClass.expectedImageDataPresent():
print(
"Error: Images specified in input header file are not present"
)
foundErr = True
if foundErr:
arch = self.findAssociatedArchive(inputDIR, dir, archList)
outFileList.write(arch + '\n')
shutil.rmtree(dir)
outFileList.flush()
outFileList.close()
def selectFiles2Keep(self, dupHdrs, headersDIR, dupArchs, useArchs,
selectVersion, archivesDIR, cpArchives2DIR):
"""
"""
if useArchs and (len(dupHdrs) != len(dupArchs)):
raise Exception(
"If using archives there should be the same number of headers and archives."
)
selKey = ""
if selectVersion == 'RANDOM':
selKey = random.choice(list(dupHdrs.keys()))
elif selectVersion == 'LANDSAT':
sensorFact = ARCSISensorFactory()
first = True
selKey = ""
selKeyGenTime = None
for baseName in dupHdrs:
print(baseName)
filePrefix3 = baseName[:3]
filePrefix4 = baseName[:4]
sensor = ""
if filePrefix3 == 'LM1' or filePrefix4 == 'LM01':
sensor = 'ls1'
elif filePrefix3 == 'LM2' or filePrefix4 == 'LM02':
sensor = 'ls2'
elif filePrefix3 == 'LM3' or filePrefix4 == 'LM03':
sensor = 'ls3'
elif filePrefix3 == 'LM4' or filePrefix4 == 'LM04':
sensor = 'ls4mss'
elif filePrefix3 == 'LM5' or filePrefix4 == 'LM05':
sensor = 'ls5mss'
elif filePrefix3 == 'LT4' or filePrefix4 == 'LS04' or filePrefix4 == 'LE04' or filePrefix4 == 'LT04':
sensor = 'ls4tm'
elif filePrefix3 == 'LT5' or filePrefix4 == 'LS05' or filePrefix4 == 'LE05' or filePrefix4 == 'LT05':
sensor = 'ls5tm'
elif filePrefix3 == 'LE7' or filePrefix4 == 'LS07' or filePrefix4 == 'LE07' or filePrefix4 == 'LT07':
sensor = 'ls7'
elif filePrefix3 == 'LC8' or filePrefix4 == 'LS08' or filePrefix4 == 'LC08':
sensor = 'ls8'
else:
raise ARCSIException(
"Sensor was not recognised for file: \"" + baseName +
"\"")
hdrFullPath = os.path.join(headersDIR, dupHdrs[baseName])
sensorClass = sensorFact.getSensorClassFromName(
sensor, False, None)
sensorClass.extractHeaderParameters(hdrFullPath, "")
cKeyTime = sensorClass.fileDateObj
if first:
selKeyGenTime = cKeyTime
selKey = baseName
first = False
elif cKeyTime > selKeyGenTime:
selKeyGenTime = cKeyTime
selKey = baseName
else:
raise Exception("Don't know the selection method.")
for baseName in dupHdrs:
if not selKey == baseName:
fullPath2Del = os.path.join(headersDIR, baseName)
if os.path.exists(fullPath2Del):
print("Deleting: " + fullPath2Del)
shutil.rmtree(fullPath2Del)
if useArchs:
archFile = os.path.join(archivesDIR, dupArchs[baseName])
archFileMV = os.path.join(cpArchives2DIR,
dupArchs[baseName])
if os.path.isfile(archFile):
print("Moving: " + archFile)
shutil.move(archFile, archFileMV)
default=10,
help="A limit on the number of tiles which can be downloaded.")
parser.add_argument("-t",
"--tmp",
type=str,
required=True,
help='''A temporary directory to do work in.''')
# Call the parser to parse the arguments.
args = parser.parse_args()
output_img = os.path.abspath(args.output)
tmp_dir = os.path.abspath(args.tmp)
if not os.path.exists(tmp_dir):
os.mkdir(tmp_dir)
sensorFact = ARCSISensorFactory()
sensor_cls_obj = sensorFact.getSensorClassFromName(args.sensor, False,
None)
sensor_cls_obj.extractHeaderParameters(args.inputheader, None)
image_bbox_latlon = sensor_cls_obj.getBBOXLatLon()
bounds_ext = (image_bbox_latlon[0] - args.buffer,
image_bbox_latlon[2] - args.buffer,
image_bbox_latlon[1] + args.buffer,
image_bbox_latlon[3] + args.buffer)
print(bounds_ext)
try:
elevation.clip(bounds=bounds_ext,
output=output_img,
max_download_tiles=args.limit,
cache_dir=tmp_dir)
def buildLookUp(self, inputDIR, headerEnding, outputFile, sensorStr, archivesDIR=None):
inputDIR = os.path.abspath(inputDIR)
hdrList = self.getListOfFiles(inputDIR, headerEnding)
archLUT = dict()
if not archivesDIR is None:
archPaths = self.getListOfArchives(archivesDIR)
for arch in archPaths:
archBaseName = os.path.basename(arch).split(".")[0]
archLUT[archBaseName] = arch
fileDict = dict()
sensorFact = ARCSISensorFactory()
duplicate = False
for fileHdr in hdrList:
print("Processing :", fileHdr)
sensorOUT = sensorStr
if sensorOUT == 'LANDSAT':
basefilename = os.path.basename(fileHdr)
filePrefix3 = basefilename[:3]
filePrefix4 = basefilename[:4]
if filePrefix3 == 'LM1' or filePrefix4 == 'LM01':
sensorOUT = 'ls1'
elif filePrefix3 == 'LM2' or filePrefix4 == 'LM02':
sensorOUT = 'ls2'
elif filePrefix3 == 'LM3' or filePrefix4 == 'LM03':
sensorOUT = 'ls3'
elif filePrefix3 == 'LM4' or filePrefix4 == 'LM04':
sensorOUT = 'ls4mss'
elif filePrefix3 == 'LM5' or filePrefix4 == 'LM05':
sensorOUT = 'ls5mss'
elif filePrefix3 == 'LT4' or filePrefix4 == 'LS04' or filePrefix4 == 'LE04' or filePrefix4 == 'LT04':
sensorOUT = 'ls4tm'
elif filePrefix3 == 'LT5' or filePrefix4 == 'LS05' or filePrefix4 == 'LE05' or filePrefix4 == 'LT05':
sensorOUT = 'ls5tm'
elif filePrefix3 == 'LE7' or filePrefix4 == 'LS07' or filePrefix4 == 'LE07' or filePrefix4 == 'LT07':
sensorOUT = 'ls7'
elif filePrefix3 == 'LC8' or filePrefix4 == 'LS08' or filePrefix4 == 'LC08':
sensorOUT = 'ls8'
else:
raise ARCSIException("Sensor was not recognised for file: \"" + fileHdr + "\"")
duplicate = False
sensorClass = sensorFact.getSensorClassFromName(sensorOUT, False, None)
sensorClass.extractHeaderParameters(fileHdr, "")
outBaseName = sensorClass.generateOutputBaseName()
if outBaseName in fileDict:
duplicate = True
fileHdr = fileHdr.replace(inputDIR, "")
if (fileHdr[0] == '/') or (fileHdr[0] == '\\'):
fileHdr = fileHdr[1:]
tmpList = {"Header" : fileHdr}
if not archivesDIR is None:
for baseArchName in archLUT:
if fileHdr.count(baseArchName) > 0:
tmpList['Archive'] = os.path.basename(archLUT[baseArchName])
break
if not duplicate:
fileDict[outBaseName] = tmpList
else:
if not 'Duplicates' in fileDict[outBaseName]:
fileDict[outBaseName]['Duplicates'] = []
fileDict[outBaseName]['Duplicates'].append(tmpList)
with open(outputFile, 'w') as outfile:
json.dump(fileDict, outfile, sort_keys=True, indent=4, separators=(',', ': '), ensure_ascii=False)
def runSort2RowPath(self, inputDir, outputDir, noDIRStruct, userInteract):
print("Sorting into ROW / PATH directories...")
arcsiUtils = ARCSIUtils()
inputDir = os.path.abspath(inputDir)
outputDir = os.path.abspath(outputDir)
if not os.path.isdir(inputDir):
raise ARCSIException(
"The input directory specified does not exist!")
if not os.path.isdir(outputDir):
raise ARCSIException(
"The output directory specified does not exist!")
inputTmpFiles = os.listdir(inputDir)
inputLandsatDIR = []
# Navigate the directory tree
for fileName in inputTmpFiles:
filePath = os.path.join(inputDir, fileName)
if os.path.isdir(filePath):
inputLandsatDIR.append(filePath)
foundRowPath = dict()
for inLandsatDIR in inputLandsatDIR:
lsMTLFile = arcsiUtils.findFile(inLandsatDIR, '*MTL.txt')
basefilename = os.path.basename(lsMTLFile)
filePrefix3 = basefilename[:3]
filePrefix4 = basefilename[:4]
sensorFact = ARCSISensorFactory()
sensorClass = None
if filePrefix3 == 'LM1' or filePrefix4 == 'LM01':
sensorClass = sensorFact.getSensorClassFromName(
'ls1', False, None)
elif filePrefix3 == 'LM2' or filePrefix4 == 'LM02':
sensorClass = sensorFact.getSensorClassFromName(
'ls2', False, None)
elif filePrefix3 == 'LM3' or filePrefix4 == 'LM03':
sensorClass = sensorFact.getSensorClassFromName(
'ls3', False, None)
elif filePrefix3 == 'LM4' or filePrefix4 == 'LM04':
sensorClass = sensorFact.getSensorClassFromName(
'ls4mss', False, None)
elif filePrefix3 == 'LM5' or filePrefix4 == 'LM05':
sensorClass = sensorFact.getSensorClassFromName(
'ls5mss', False, None)
elif filePrefix3 == 'LT4' or filePrefix4 == 'LS04' or filePrefix4 == 'LE04' or filePrefix4 == 'LT04':
sensorClass = sensorFact.getSensorClassFromName(
'ls4tm', False, None)
elif filePrefix3 == 'LT5' or filePrefix4 == 'LS05' or filePrefix4 == 'LE05' or filePrefix4 == 'LT05':
sensorClass = sensorFact.getSensorClassFromName(
'ls5tm', False, None)
elif filePrefix3 == 'LE7' or filePrefix4 == 'LS07' or filePrefix4 == 'LE07' or filePrefix4 == 'LT07':
sensorClass = sensorFact.getSensorClassFromName(
'ls7', False, None)
elif filePrefix3 == 'LC8' or filePrefix4 == 'LS08' or filePrefix4 == 'LC08':
sensorClass = sensorFact.getSensorClassFromName(
'ls8', False, None)
else:
print("Sensor was not recognised for file: " + file)
sensorClass.extractHeaderParameters(lsMTLFile, None)
outdir = 'r' + str(sensorClass.row) + 'p' + str(sensorClass.path)
outputFileDIR = os.path.join(outputDir, outdir)
if not (outdir in foundRowPath):
foundRowPath[outdir] = True
self.createDIRStruct(outputFileDIR, noDIRStruct, False)
mv2DIR = outputFileDIR
if not noDIRStruct:
mv2DIR = os.path.join(outputFileDIR, 'Inputs')
print("Moving: " + inLandsatDIR)
print("To: " + mv2DIR)
self.moveFile(inLandsatDIR, mv2DIR, userInteract)