def downloadScene(sceneID):
# call download_landsat_data_by_sceneid.php
in_dir = os.getcwd()
os.chdir(LSF.path_projects + '/dataexchange')
print os.getcwd()
# download the scene data
errcode=subprocess.call(["php", "download_landsat_data_by_sceneid.php", sceneID])
os.chdir(in_dir)
if errcode:
raise RuntimeError(errcode)
# check to make sure the download succeeded
inNewSceneTar = os.path.join(LSF.tarStorage, sceneID+".tar.gz")
print inNewSceneTar
err=localLib.validTar(inNewSceneTar)
if err:
os.remove(inNewSceneTar)
raise DownloadError(err)
def extractedTar(quadsceneID):
sceneID=quadsceneID[:-2]
# guarantee that level 1 product tar file for sceneID now exists in /lsfdata/eros_data/extractedTars
tar(sceneID)
# if /lsfdata/extractedTars/sceneID doesn't contain the level 1 product band files, extract them from the tar file
#
if not(re.search(sceneID, ' '.join(glob.glob(os.path.join(LSF.tiffsStorage, '*'))))):
# make sure that the existing tar is valid
existingTar=os.path.join(LSF.tarStorage, sceneID+".tar.gz")
err=localLib.validTar(existingTar)
if err:
raise Exception(err)
# for now use checkExisting. change to use tarHandling class when completed
extractedPath = landsatFactTools_GDAL.checkExisting(existingTar, LSF.tiffsStorage)
rasterAnalysis_GDAL.runFmask(extractedPath,LSF.fmaskShellCall)
# get DN min number from each band in the scene and write to database
dnminExists = landsatFactTools_GDAL.checkForDNminExist(extractedPath) # May not be needed in final design, used during testing
if dnminExists == False:
dnMinDict = rasterAnalysis_GDAL.getDNmin(extractedPath)
# fix bug introduced by me in commit de5df07c4ca3ff71ae4d7da27b6018fe1bc2df04
landsatFactTools_GDAL.writeDNminToDB(dnMinDict,extractedPath)
infile = sys.argv[1]
with open(infile) as inf:
for line in inf:
m=re.search('L.*?\.tar\.gz', line)
if m:
runList.append(m.group())
runList.sort(key=lambda tar: tar[9:16])
os.chdir(tarStorage)
for tar in runList:
try:
# set tar file to analyze
if not tar[-7:] == '.tar.gz':
print "incorrect file type"
raise RuntimeError("Not a tarball: "+tar)
err=localLib.validTar(tar)
if err:
os.remove(tar)
landsatFactTools_GDAL.retry(1, 4, landsatFactTools_GDAL.DownloadError,landsatFactTools_GDAL.downloadScene, tar[:-7])
# all paths are now imported from LSF.py DM - 5/10/2016
# =========================================================================
# sets full path for the tarfile to be analyzed
inNewSceneTar = os.path.join(tarStorage, tar)
# check to see if the tar file has been extracted, if not extract the files
print "Checking for: "+tar
extractedPath = landsatFactTools_GDAL.checkExisting(inNewSceneTar, tiffsStorage)
# run Fmask
# jdm 4/22/15: after spending a couple of days trying to get FMASK installed on cloud4
# I have not been able to get it to work. Therefore, for now I am commenting this out
# rasterAnalysis_GDAL.runFmask(extractedPath,Fmaskexe) #BM's original
print extractedPath
#! /usr/bin/python
#**********************************************************************
# Description:
# Check that the input tar is valid and has all the required files.
# Returns a return code of 1 for True and 0 for False. Removes the tar if false.
# A negative return code indicates an error.
#
# Arguments:
# 0 - Input file (inTar)
#
#**********************************************************************
import sys, traceback, localLib, logging
inTar = sys.argv[1]
try:
missing = localLib.validTar(inTar)
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = "PYTHON ERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \
str(sys.exc_type)+ ": " + str(sys.exc_value) + "\n"
logging.error(pymsg)
sys.exit(-1)
if missing:
localLib.removeTar(inTar)
sys.exit(0)
else:
sys.exit(1)
# Returns a return code of 1 for True and 0 for False. Removes the zip if false.
# Returns 0 if any of the gz's were invalid, 1 if they were all valid.
# A negative return code indicates an error.
#
# Arguments:
# 0 - Input file (inTarDir)
#
#**********************************************************************
import os, sys, traceback, logging, glob, localLib
inTarDir = sys.argv[1]
try:
errCode = 1
os.chdir(inTarDir)
for tar in glob.glob('*.gz'):
print tar
err = localLib.validTar(tar)
if err:
errCode = 0
localLib.removeTar(tar)
except:
tb = sys.exc_info()[2]
tbinfo = traceback.format_tb(tb)[0]
pymsg = "PYTHON ERRORS:\nTraceback Info:\n" + tbinfo + "\nError Info:\n " + \
str(sys.exc_type)+ ": " + str(sys.exc_value) + "\n"
logging.error(pymsg)
errCode = -1
sys.exit(errCode)