def downloadScene(sceneID): # call download_landsat_data_by_sceneid.php in_dir = os.getcwd() os.chdir(LSF.path_projects + '/dataexchange') print os.getcwd() # download the scene data errcode=subprocess.call(["php", "download_landsat_data_by_sceneid.php", sceneID]) os.chdir(in_dir) if errcode: raise RuntimeError(errcode) # check to make sure the download succeeded inNewSceneTar = os.path.join(LSF.tarStorage, sceneID+".tar.gz") print inNewSceneTar err=localLib.validTar(inNewSceneTar) if err: os.remove(inNewSceneTar) raise DownloadError(err)
def extractedTar(quadsceneID): sceneID=quadsceneID[:-2] # guarantee that level 1 product tar file for sceneID now exists in /lsfdata/eros_data/extractedTars tar(sceneID) # if /lsfdata/extractedTars/sceneID doesn't contain the level 1 product band files, extract them from the tar file # if not(re.search(sceneID, ' '.join(glob.glob(os.path.join(LSF.tiffsStorage, '*'))))): # make sure that the existing tar is valid existingTar=os.path.join(LSF.tarStorage, sceneID+".tar.gz") err=localLib.validTar(existingTar) if err: raise Exception(err) # for now use checkExisting. change to use tarHandling class when completed extractedPath = landsatFactTools_GDAL.checkExisting(existingTar, LSF.tiffsStorage) rasterAnalysis_GDAL.runFmask(extractedPath,LSF.fmaskShellCall) # get DN min number from each band in the scene and write to database dnminExists = landsatFactTools_GDAL.checkForDNminExist(extractedPath) # May not be needed in final design, used during testing if dnminExists == False: dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath) # fix bug introduced by me in commit de5df07c4ca3ff71ae4d7da27b6018fe1bc2df04 landsatFactTools_GDAL.writeDNminToDB(dnMinDict,extractedPath)
infile = sys.argv[1] with open(infile) as inf: for line in inf: m=re.search('L.*?\.tar\.gz', line) if m: runList.append(m.group()) runList.sort(key=lambda tar: tar[9:16]) os.chdir(tarStorage) for tar in runList: try: # set tar file to analyze if not tar[-7:] == '.tar.gz': print "incorrect file type" raise RuntimeError("Not a tarball: "+tar) err=localLib.validTar(tar) if err: os.remove(tar) landsatFactTools_GDAL.retry(1, 4, landsatFactTools_GDAL.DownloadError,landsatFactTools_GDAL.downloadScene, tar[:-7]) # all paths are now imported from LSF.py DM - 5/10/2016 # ========================================================================= # sets full path for the tarfile to be analyzed inNewSceneTar = os.path.join(tarStorage, tar) # check to see if the tar file has been extracted, if not extract the files print "Checking for: "+tar extractedPath = landsatFactTools_GDAL.checkExisting(inNewSceneTar, tiffsStorage) # run Fmask # jdm 4/22/15: after spending a couple of days trying to get FMASK installed on cloud4 # I have not been able to get it to work. Therefore, for now I am commenting this out # rasterAnalysis_GDAL.runFmask(extractedPath,Fmaskexe) #BM's original print extractedPath
#! /usr/bin/python #********************************************************************** # Description: # Check that the input tar is valid and has all the required files. # Returns a return code of 1 for True and 0 for False. Removes the tar if false. # A negative return code indicates an error. # # Arguments: # 0 - Input file (inTar) # #********************************************************************** import sys, traceback, localLib, logging inTar = sys.argv[1] try: missing = localLib.validTar(inTar) except: tb = sys.exc_info()[2] tbinfo = traceback.format_tb(tb)[0] pymsg = "PYTHON ERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \ str(sys.exc_type)+ ": " + str(sys.exc_value) + "\n" logging.error(pymsg) sys.exit(-1) if missing: localLib.removeTar(inTar) sys.exit(0) else: sys.exit(1)
# Returns a return code of 1 for True and 0 for False. Removes the zip if false. # Returns 0 if any of the gz's were invalid, 1 if they were all valid. # A negative return code indicates an error. # # Arguments: # 0 - Input file (inTarDir) # #********************************************************************** import os, sys, traceback, logging, glob, localLib inTarDir = sys.argv[1] try: errCode = 1 os.chdir(inTarDir) for tar in glob.glob('*.gz'): print tar err = localLib.validTar(tar) if err: errCode = 0 localLib.removeTar(tar) except: tb = sys.exc_info()[2] tbinfo = traceback.format_tb(tb)[0] pymsg = "PYTHON ERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \ str(sys.exc_type)+ ": " + str(sys.exc_value) + "\n" logging.error(pymsg) errCode = -1 sys.exit(errCode)