#prompt user signal

cont= raw_input("script at line %s, \n CONTINUE? Y/n?" %lNum)

if cont == "n":

print "stop signal received from user for image %s at line %s" %(baseName, lNum)

print "\n\n -->quitting script on user input"

multifolds=(iDir +"/"+ tile for tile in tiles)

bnlist=[]

rlist=[]

#search for tiff files in folder

for i in multifolds:

for r,d,files in os.walk(i):

for f in files:

if f.endswith(".tif"):

bnlist.append(f)

rlist.append(os.path.join(r,f))

finlist=[bnlist, rlist]

print "output of function 1.1 srcImg:\n\n %s"

for i in finlist[0:50]:

print i

#comparing image lists

RlistNP=[y for y in Orlist[1] if os.path.basename(y) not in Outlist[0]]

#if sublist is of length 0 exit right now

if len(RlistNP) == 0:

print "no image to correct.\n\n -->quitting script on trigger"

quit()

else:

logstr2= "\nfound %d unprocessed images" %(len(RlistNP))

Lfile.write(logstr2)

print logstr2

print "\nthe following images will be processed"

print "output of function checkList:\n\n"

for i in RlistNP:

print os.path.basename(i)

global baseName

global orImg

global extImg

global imgCrs

baseName = os.path.basename(ipat)[0:-4]

orImg = QgsRasterLayer(ipat, baseName)

if not orImg.isValid():

print "\nLayer failed to load!\n\n -->quitting script on image %s" % baseName

raise SystemExit

# image extension as string ADDED new extension to include only parts overlapping study site

oldExt=orImg.extent()

stExt=QgsVectorLayer(stArea, "stArea", "ogr").extent()

newExt=oldExt.intersect(stExt)

extImg=newExt.toString().replace(" : ",",")

#check of reference system and reprojection

imgCrs= orImg.crs()

#output dictionary

mdImg={}

mdImg["baseName"]=baseName

mdImg["orImg"]=orImg

mdImg["extImg"]=extImg

mdImg["imgCrs"]=imgCrs

print "output of function 2 imgMd:\n\n %s" %str(mdImg)

directory=temp1+"/"+baseName+"/"

if not os.path.exists(directory):

os.makedirs(directory)

print "output of function 2.1 makeWod, %s" %directory

demCrs=QgsRasterLayer(dem).crs()

print "\nImage CRS :%s\n dem crs:%s" %(imgCrs.description(), demCrs.description())

if demCrs != imgCrs:

print "\nreprojecting DEM to %s" %(imgCrs.description())

ldem=QgsRasterLayer(dem)

extdem="%f,%f,%f,%f" %(ldem.extent().xMinimum(),\

ldem.extent().xMaximum(),\

ldem.extent().yMinimum(),\

ldem.extent().yMaximum())

crsStr=imgCrs.authid()

newdem=p.runalg("gdalogr:warpreproject",ldem,"",crsStr,"",30,0,False,extdem,"",5,4,75,6,1,False,0,False,"",None)

LnDem=QgsRasterLayer(newdem['OUTPUT'])

if LnDem.isValid():

print "dem reprojected correctly"

dem=newdem['OUTPUT']

print "output of function 3 demCheck:\n\n %s" %dem

else:

print "\nall files are in the same CRS!"

sBand={}

#band name and path

bname="band"+str(bNum)

directory=wod

path=directory+bname+".tif"

# call to gdal translate with option

gdal.Translate(path, ipat, bandList=[bNum])

if not QgsRasterLayer(path).isValid():

print "problem saving single band %s!\n\n -->quitting script on image %s, function bndSep" %(bNum,baseName)

quit()

print " output of function 4 bndSep:\n\n %s" %path

# preparation parameters for 6S

#path to parameter file

band="band"+str(bNum)

Spath=str(wod+band+"_.atcorParam.txt")

#Writing paramater file

Sfile=open(Spath, "w")

#FIRST LINE : satellite type

firstLine="25\n"

Sfile.write(firstLine)

#SECOND LINE

#read end of basename for date and time

dateS=baseName.split("_")[2]

month=dateS[4:6]

day=dateS[6:8]

hours=dateS[9:11]

minutes=dateS[11:13]

# hours and day in decimal time

dmin=round(100*float(minutes)/60,0)

dtime=str(hours)+"."+str(dmin/100)[2:]

#long lat (centre point) in WGS84

lon=orImg.extent().xMinimum()+((orImg.extent().xMaximum()-orImg.extent().xMinimum())/2)

lat=orImg.extent().yMinimum()+((orImg.extent().yMaximum()-orImg.extent().yMinimum())/2)

# transform long and lat in WGS 84

crsDest=QgsCoordinateReferenceSystem(4326)

xform = QgsCoordinateTransform(orImg.crs(), crsDest)

ncoord=xform.transform(QgsPoint(lon,lat))

#print secondline

secondLine=str(month)+" "+str(day)+" "+str(dtime)+" "+str(ncoord.x())+" "+str(ncoord.y())+"\n"

Sfile.write(secondLine)

#THIRD LINE of parameter file athmospheric model (1) continental

if month < 3 or month > 10:

atmMod=3

else:

atmMod=2

thirdLine="%s \n" %atmMod

#print third line

Sfile.write(thirdLine)

#FOURTH LINE: 1 continental model

fourthLine="1\n"

Sfile.write(fourthLine)

#FIFTH LINE: Aerosol concentration model (visibility)

fifthLine="-1\n"

Sfile.write(fifthLine)

#SIXTH LINE: Aerosol concentration model (visibility)

sixthLine="0\n"

Sfile.write(sixthLine)

#SEVENTH LINE (average altitude in negative km)

# missing: actual calculation of average altitude avgAlt

avgAlt=1400

avgDem=-1*float(avgAlt)/1000

seventhLine=str(avgDem)+"\n"

Sfile.write(seventhLine)

# EIGHTH LINE: sensor altitude

eighthLine="-1000\n"

Sfile.write(eighthLine)

# F : sensor band code

bcodes=range(166, 179)

ninthLine=str(bcodes[bNum-1])+"\n"

Sfile.write(ninthLine)

#output

Sfile.close()

print "output of function 5 makePar is:\n\n"

print "1st line of 6S parameter file (SATELLITE TYPE) is %s\n" %firstLine

print "2nd line of 6S parameter file is (month, day, hh.ddd, long, lat):\n %s" %secondLine

print "3rd line of 6S parameter file is (Athmospheric model):\n %s" %thirdLine

print "4th line of 6S parameter file is (Aereosol model):\n %s" %fourthLine

print "5th and 6th line of 6S parameter file is (Visibility model):\n %s and %s" %(fifthLine, sixthLine)

print "7th and 8th lines of 6S parameter file (target and sensor altitude are:\n %s %s " %(seventhLine, eighthLine)

print "9th line of 6S parameter file (band code) is %s " %(ninthLine)

#range of values

ds = gdal.Open(singImg)

myarray = np.array(ds.GetRasterBand(1).ReadAsArray())

pMin=np.nanmin(myarray)

pMax=np.nanmax(myarray)

pRange=str(pMin)+","+str(pMax)

print "output of function 5.1 pixRange is:"

print 'pixel range is %s' %(pRange)

ds=None

# double check of input files

if not QgsRasterLayer(singImg).isValid():

print "problem with input image before atmospheric correction!\n\n -->quitting script on image %s" % baseName

#stopGo(338, baseName)

print "launching athmospheric correction on image "+baseName[-10:]

#path to corrected image

bname="band"+str(bNum)

corPath=wod+"corr"+bname+".tif"

corrImg=p.runalg("grass7:i.atcorr",singImg,pRange,None, None,par,pRange,False,True,False,False,extImg,0,corPath)

if not QgsRasterLayer(corPath).isValid():

print "problem correcting the image!\n\n -->quitting script on image %s" % baseName

#stopGo(346, baseName)

if corrImg['output'] != corPath:

print "problem with corrected image location"

else:

print "output of function 6 atCorr is: \n\n"

print corPath

fStr="(B-A)/(B+A)"

ndPath=wod+"ndvi.tif"

ndPath2=wod+"ndviClip.tif"

nd=p.runalg("gdalogr:rastercalculator",Red,"1",Nir,"1",None,"1",None,"1",None,"1",None,"1",fStr,"",5,"",ndPath)

# clip raster to study site area

# prepare extension in gdal_translate format (xmin,ymin,xmax,ymax

xmin,ymin,xmax,ymax=extImg.split(",")

ext="%s %s %s %s" %(xmin,ymax,xmax,ymin)

cmd="gdal_translate -ot Float32 -projwin %s %s %s" %(ext, ndPath, ndPath2)

os.system(cmd)

if not QgsRasterLayer(ndPath2).isValid():

print "problem cerating NDVI!\n\n -->quitting script on image %s" % baseName

raise SystemExit

else:

print "output of function 7 ndCalc is: \n\n"

print ndPath2

fStr="((B-A)*(1+0. 7))/(B+A+0.7)"

ndPath=wod+"SAVI.tif"

nd=p.runalg("gdalogr:rastercalculator",Red,"1",Nir,"1",None,"1",None,"1",None,"1",None,"1",fStr,"",5,"",ndPath)

if not QgsRasterLayer(ndPath).isValid():

print "problem creating SAVI!\n\n -->quitting script on image %s" % baseName

#stopGo(367, baseName)

else:

print "output of function 7 ndCalc is: \n\n"

print ndPath

fStr="(2*B+1-sqrt (((2*B+1)^2)-(8*(B-A)))/2"

ndPath=wod+"MSAVI2.tif"

nd=p.runalg("gdalogr:rastercalculator",Red,"1",Nir,"1",None,"1",None,"1",None,"1",None,"1",fStr,"",5,"",ndPath)

if not QgsRasterLayer(ndPath).isValid():

print "problem creating MSAVI2!\n\n -->quitting script on image %s" % baseName

raise SystemExit

else:

print "output of function 7 ndCalc is: \n\n"

print ndPath

# find path to cloudmask

year=baseName.split("_")[2][0:4]

tile=baseName.split("_")[5]

mainDir=os.path.join(clPath, tile, year, baseName+".SAFE")

if not os.path.exists(mainDir):

print "problem finding cloud mask folder"

raise SystemExit

# look for cloudmask in remote folder

for r, d, files in os.walk(mainDir):

for f in files :

if f.endswith("B00.gml"):

clMpath=os.path.join(r,f)

cpv=wod+"cloudVec.gml"

shutil.copy2(clMpath, cpv)

# mask raster using vector

cloudRes=QgsRasterLayer(img).rasterUnitsPerPixelX()

cloudEx=extImg

rMaskPath=wod+"clMaskNdvi.tif"

print "starting cloud masking with the following parameters:"

print "parameter 1: cloudMask vector: {}\n" .format(cpv)

print "parameter 2: Raster image to mask:{} \n" .format(img)

print "parameter 6: extension: {}\n" .format(cloudEx)

print "parameter 7: resolution: {}\n" .format(cloudRes)

print "parameter 10: output location: {} \n" .format(rMaskPath)

x=datetime.datetime.now()

p.runalg("grass7:r.mask.vect", cpv, img,"","",True, cloudEx, cloudRes, -1, 0.00001,rMaskPath)

y=datetime.datetime.now()

c=y-x

d=divmod(c.days*86400+c.seconds, 60)

print "finished cloud masking in %f minutes and %f seconds" %(d[0], d[1])

if not QgsRasterLayer(rMaskPath).isValid():

print "problem masking image %s function 8 clMask" %baseName

raise SystemExit

else:

print"output of function 8 clMask is:\n %s" %rMaskPath

year=baseName.split("_")[2][0:4]

tile=baseName.split("_")[5]

mainDir=os.path.join(clPath, tile, year, baseName+".SAFE")

if not os.path.exists(mainDir):

print "problem finding cloud mask folder"

raise SystemExit

# look for cloudmask in remote folder

for r, d, files in os.walk(mainDir):

for f in files:

if f.endswith("B00.gml"):

clMpath=os.path.join(r,f)

global clMpath

cpv=wod+"cloudVec.shp"

if not QgsVectorLayer(clMpath, "cl", "ogr").isValid():

print "problem with original cloud mask or no clouds in the image"

return img

#translate gml to shapefile

ogr2ogr.main(["","-f", "ESRI Shapefile", "-s_srs", "EPSG:32630", cpv, clMpath])

if not QgsVectorLayer(cpv, "cl", "ogr").isValid():

print "cloud vector translation didn't go well"

raise SystemExit

else:

print "cloud vector translation did work!!"

# rasterize cloud mask

# parameters to rasterize

cloudRes=QgsRasterLayer(img).rasterUnitsPerPixelX()

cloudRast=wod+"clMask.tif"

#rasterization

x=datetime.datetime.now()

cmd="gdal_rasterize -burn 10 -a_srs %s -te %s -tr %s %s %s %s" %(imgCrs.authid(), extImg.replace(","," "), cloudRes, cloudRes, cpv, cloudRast)

print cmd

os.system(cmd)

#change nodata value

cmd="gdal_edit.py -unsetnodata %s" %(cloudRast)

os.system(cmd)

print "check nodatavalues in %s" %(cloudRast)

#cloudRast2=wod+"clMaskNoData.tif"

#p.runalg("grass7:r.null", cloudRast,"",1,False,False,False,False,False, extImg,cloudRes, cloudRast2)

if not QgsRasterLayer(cloudRast).isValid():

print "Cloud raster %s has a problem \n" %(cloudRast)

# Mask out cloudy pixels

rMaskPath=wod+"clMaskNDVI.tif"

cmd="gdal_calc.py -A %s -B %s --outfile=%s --calc='A*(B<9)' " %(img, cloudRast, rMaskPath)

print(cmd)

os.system(cmd)

y=datetime.datetime.now()

c=y-x

d=divmod(c.days*86400+c.seconds, 60)

print "finished cloud masking in %f minutes and %f seconds" %(d[0], d[1])

if not QgsRasterLayer(rMaskPath).isValid():

print "problem masking image %s function 8 clMask" %baseName

raise systemExit

else:

print"output of function 8 clMask is:\n %s" %rMaskPath

print "starting makeOut"

tile=baseName.split("_")[5]

outDir=os.path.join(outd, tile)

outPath=os.path.join(outDir, baseName+".tif")

if not os.path.exists(outDir):

os.makedirs(outDir)

shutil.copy2(Img, outPath)

if not QgsRasterLayer(outPath).isValid():

print "problem saving final image %s\n\n -->quitting script on function 100.2 makeOut"

raise SystemExit

print "output of function 100.2 makeOut is:\n %s" %outPath

print "\n\n***START PROCESSING IMAGE: %s***" %os.path.basename(ipat)

## call function to get image metadata

mdDic=imgMd(ipat, stArea)

#output should be dictionary

## function to create working directory for this image

wod=makeWod(baseName, temp1)

global wod

print "wod is %s" %wod

#output is working folder path

## function to check for dem crs

#parameters

global dem

dem=demCheck(dem, imgCrs, wod)

#raise SystemExit

#output is path to (new) dem

##------- RED IMAGE

## function to seprate multilayer image in single band (RED)

bNum=4

Red=bndSep(ipat, bNum,baseName, wod)

#output is path to RED band geotiff

## function to create parameters for athm. corr. in single band image (RED)

orImg=mdDic ["orImg"]

redPar=makePar(orImg, baseName, bNum, wod)

## function to calculate pixel range in band (RED)

redRange=pixRange(Red, wod)

## function to perform athm. corr. on single band image (RED)

redCor=atCorr(Red, baseName, redPar, redRange, bNum, extImg, wod )

# output should be path to corrected image (RED)

print "corrected image is %s" %redCor

# stopGo(478, baseName)

###------- NIR IMAGE

## function to seprate multilayer image in single band (NIR)

bNum=8

Nir=bndSep(ipat, bNum, baseName, wod)

#output is path to NIR band geotiff

## function to create parameters for athm. corr. in single band image (NIR)

nirPar=makePar(orImg, baseName, bNum, wod)

## function to calculate pixel range in band (NIR)

nirRange=pixRange(Nir, wod)

## function to perform athm. corr. on single band image (NIR)

nirCor=atCorr(Nir, baseName, nirPar, nirRange, bNum, extImg, wod )

print "corrected image is %s" %nirCor

# output should be path to corrected image (NIR)

# stopGo(498, baseName)

## Function to calculate NDVI

nd=ndCalc(redCor, nirCor, wod)

print "ndvi should be in %s" %nd

# output should be path to corrected image (NIR)

# stopGo(504, baseName)

## function to mask ndvi using cloudmask of image

msNdvi=clMask2(nd, clPath, baseName, wod)

# raise SystemExi

#stopGo(558, baseName)

##function to export final image (masked NDVI)

fin=makeOut(msNdvi, baseName, outd)

#if working files are not needed anymore

#cleanUp(wod, baseName)

#stopGo(514, baseName)