def dlthumb(url, jpgdir, *args, **kwargs): # This downloads thumbnails from the USGS
global errorsfound
basename = os.path.basename(url)
f = os.path.join(jpgdir, basename)
tries = 1
downloaded = False
print('Downloading {} to {}'.format(basename, jpgdir))
while not downloaded and tries < 6:
print('Download attempt {} of 5.'.format(tries))
try:
url = urlopen(dlurl)
urlretrieve(dlurl, filename = f)
if url.length == os.stat(f).st_size:
downloaded = True
else:
print('Error downloading, retrying.')
tries += 1
except urllib.error.URLError as e:
print(e.reason)
ieo.logerror(dlurl, e.reason, errorfile = errorfile)
errorsfound = True
if tries == 6:
ieo.logerror(f, 'Download error.', errorfile = errorfile)
print('Download failure: {}'.format(basename))
errorsfound = True
else:
return 'Success!'
def dlxmls(startdate, enddate, xmls, ingestdir, *args, **kwargs): # This downloads queried XML files
#pathrows = [[207, 208, 21, 21],[205, 209, 22, 24]]
global errorsfound
tries = 1
downloaded = False
for x, p in zip(xmls, pathrows):
print('Downloading {} to: {}'.format(x, ingestdir))
xml = os.path.join(ingestdir, x)
if os.access(xml, os.F_OK):
print('Backing up current xml file.')
shutil.move(xml, '{}.{}.bak'.format(xml, today.strftime('%Y%m%d-%H%M%S')))
urlname = 'http://earthexplorer.usgs.gov/EE/InventoryStream/pathrow?start_path={}&end_path={}&start_row={}&end_row={}&sensor_name=LANDSAT_COMBINED_C1&start_date={}&end_date={}'.format(p[0], p[1], p[2], p[3], startdate, enddate) #&cloud_cover = 100&seasonal = False&aoi_entry=path_row&output_type=unknown
tries = 1
downloaded = False
while not downloaded and tries < 6:
print('Download attempt {} of 5.'.format(tries))
try:
# url = urllib.request.urlopen(urlname)
urlretrieve(urlname, xml) # filename=xml
downloaded = True
except URLError as e:
print(e.reason)
ieo.logerror(urlname, e.reason, errorfile = errorfile)
errorsfound = True
tries += 1
if tries == 6:
ieo.logerror(xml, 'Download error.', errorfile = errorfile)
print('Download failure: {}'.format(x))
errorsfound = True
else:
return 'Success!'
def getscenedata(layer, localscenelist):
scenedata = {}
for feature in layer:
sceneID = feature.GetField("sceneID")
ProductID = feature.GetField('LANDSAT_PRODUCT_ID')
includescene = True
sunEl = feature.GetField("sunElevation")
sensor = feature.GetField("SensorID")
acqDateval = feature.GetField("acquisitionDate")
try:
acqDate = datetime.datetime.strptime(acqDateval, '%Y/%m/%d')
datestr = acqDate.strftime('%Y%j')
except:
print(
'Error: "acqDate" field missing acquisition date data, attempting to correct.'
)
ieo.logerror(
sceneID,
'"acquisitionDate" field missing acquisition date data, attempting to correct.'
)
datestr = sceneID[9:16]
acqDate = datetime.datetime.strptime(datestr, '%Y%j')
# feature.SetField('acqDate', acqDate)
proclevel = feature.GetField("DATA_TYPE_L1")
if sceneID[2:3] == '8' and ((datestr in L8exclude) or
(sensor != 'OLI_TIRS')):
includescene = False
if sceneID[2:3] == '7' and datestr in L7exclude:
includescene = False
if sunEl: # ignore Null values
if includescene and sunEl >= args.minsunel:
SR_file = feature.GetField('Surface_reflectance_tiles')
scenedata[sceneID] = {
'LANDSAT_PRODUCT_ID': ProductID,
'acquisitionDate': acqDate,
'Path': feature.GetField("path"),
'Row': feature.GetField("row"),
'SensorID': sensor,
'cloudCoverFull': feature.GetField("cloudCoverFull"),
'CLOUD_COVER_LAND': feature.GetField("CLOUD_COVER_LAND"),
'sunElevation': sunEl,
'Surface_reflectance_tiles': SR_file,
'proclevel': proclevel
}
if SR_file and not args.usesrdir:
if os.path.isfile(SR_file):
localscenelist.append(os.path.basename(SR_file)[:16])
return scenedata, localscenelist
def findmissing(l8, l47, scenedata, localscenelist, cctype):
keys = scenedata.keys()
for sceneID in keys:
if not sceneID[:16] in localscenelist:
try:
if sceneID[2:3] == '8' and any(
sceneID[9:16] in key for key in l8.keys()) and not any(
sceneID in l8[key] for key in
l8.keys()) and scenedata[sceneID][cctype] < 100.0:
print('Adding {} to Landsat 8 processing list.'.format(
sceneID))
# if not sceneID[9:16] in l8.keys() and any(sceneID[9:16] == key[9:16] for key in l8.keys()):
# l8[sceneID[9:16]] = [sceneID]
# else:
l8[sceneID[9:16]].append(sceneID)
elif sceneID[2:3] != '8' and any(
sceneID[9:16] in key
for key in l47.keys()) and not any(
sceneID in l47[key] for key in
l47.keys()) and scenedata[sceneID][cctype] < 100.0:
print('Adding {} to Landsat 4-7 processing list.'.format(
sceneID))
# if not sceneID[9:16] in l47.keys():
# l47[sceneID[9:16]] = [sceneID]
# else:
l47[sceneID[9:16]].append(sceneID)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
print('ERROR: {} {} {} {}.'.format(sceneID, exc_type, fname,
exc_tb.tb_lineno))
ieo.logerror(
sceneID, '{} {} {}'.format(exc_type, fname,
exc_tb.tb_lineno))
# sc = scenesearch(scenedata, sceneID)
# if len(sc) > 0:
# for s in sc:
# if not scenedata[s][6]:
# if s[2:3] == '8' and not s in l8:
# print('Adding %s to Landsat 8 processing list.'%s)
# l8.append(s)
# elif not s in l47:
# print('Adding %s to Landsat 4-7 processing list.'%s)
# l47.append(s)
return l8, l47
# This look finds any existing processed data
for dir in [args.outdir, os.path.join(args.outdir, 'L1G')]:
rlist = glob.glob(
os.path.join(args.outdir, '*_ref_{}.dat'.format(ieo.projacronym)))
for f in rlist:
if not 'ESA' == os.path.basename(f)[16:19]:
reflist.append(f)
# Now create the processing list
if args.infile: # This is in case a specific file has been selected for processing
if os.access(args.infile, os.F_OK) and args.infile.endswith('.tar.gz'):
print('File has been found, processing.')
filelist.append(args.infile)
else:
print('Error, file not found: {}'.format(args.infile))
ieo.logerror(args.infile, 'File not found.')
else: # find and process what's in the ingest directory
for root, dirs, files in os.walk(args.indir, onerror=None):
for name in files:
if name.endswith('.tar.gz') or name.endswith('_sr_band7.img'):
fname = os.path.join(root, name)
ssceneID = sceneidfromfilename(name)
if ssceneID:
sslist = [x for x in scenedict.keys() if ssceneID in x]
if len(sslist) > 0:
for sceneID in sslist:
if (
args.overwrite or not any(ssceneID in x
for x in reflist)
) and (
not fname in filelist
def populatelists(l8, l47, scenedata, localscenelist):
for sceneID in scenedata.keys():
acqDate = scenedata[sceneID]['acquisitionDate']
path = scenedata[sceneID]['Path']
row = scenedata[sceneID]['Row']
scenesensor = scenedata[sceneID]['SensorID']
if args.ccland:
cc = scenedata[sceneID]['CLOUD_COVER_LAND']
if not cc:
cc = 0.0
maxcc = args.maxccland
cctype = 'CLOUD_COVER_LAND'
else:
cc = scenedata[sceneID]['cloudCoverFull']
if not cc:
cc = 0.0
maxcc = args.maxcc
cctype = 'cloudCoverFull'
if cc == None:
cc = 0.0
sunEl = scenedata[sceneID]['sunElevation']
# SR = scenedata[sceneID]['Surface_reflectance_tiles']
proclevel = scenedata[sceneID]['proclevel']
try:
if (
not any(sceneID[:16] in x
for x in localscenelist) or args.ignorelocal
) and (cc <= maxcc) and (sunEl >= args.minsunel) and (
proclevel in proclevels
): # Only run this for scenes that aren't present on disk or if we choose to ignore local copies.
# if (feature.GetField("SR_path") == None or args.ignorelocal) and feature.GetField("CCFull") <= args.maxcc and feature.GetField("sunEl") >= args.minsunel:
# sceneID = feature.GetField("sceneID")
if args.landsat:
if args.landsat != int(sceneID[2:3]):
continue
if args.path:
if args.path != path:
continue
# else:
# print(path)
if args.row:
if args.row != row:
continue
# else:
# print(row)
if args.sensor:
if sensor != scenesensor:
continue
year = int(sceneID[9:13])
doy = int(sceneID[13:16])
if args.startyear or args.endyear:
if args.startyear > args.endyear:
endyear = args.startyear
startyear = args.endyear
else:
endyear = args.endyear
startyear = args.startyear
if year < startyear or year > endyear:
continue
if args.startdoy and args.enddoy: # This might be programmed later to restrict dates to specific dates/ times of year
if args.startdoy < args.enddoy:
if doy < args.startdoy or doy > args.enddoy:
continue
else:
if args.startyear:
if year == startyear and doy < args.startdoy:
continue
if args.endyear:
if year == endyear and doy > args.enddoy:
continue
if doy > endday and doy < startday:
continue
if (acqDate >= args.startdate) and (acqDate <= args.enddate):
#
print(
'Scene {}, cloud cover of {} percent, added to list.'.
format(sceneID, cc))
if not sceneID[9:16] in L7exclude and not sceneID[
2:
3] == '8': #(scenesensor == 'LANDSAT_TM' or scenesensor == 'LANDSAT_ETM' or 'LANDSAT_ETM_SLC_OFF') and
if not sceneID[9:16] in l47.keys():
l47[sceneID[9:16]] = [sceneID]
elif not sceneID in l47[sceneID[9:16]]:
l47[sceneID[9:16]].append(sceneID)
if args.allinpath:
sc = scenesearch(scenedata, sceneID, pathrowdict)
if len(sc) > 0:
for s in sc:
if not s in l47[sceneID[9:16]]:
print(
'Also adding scene {} to the processing list.'
.format(sceneID))
l47[sceneID[9:16]].append(s)
# elif scenesensor=='LANDSAT_ETM':
# l7.append(sceneID)
# elif scenesensor=='LANDSAT_ETM_SLC_OFF' and not sceneID[9:16] in L7exclude:
# l7slcoff.append(sceneID)
elif sceneID[2:3] == '8' and not sceneID[9:16] in L8exclude:
if not sceneID[9:16] in l8.keys():
l8[sceneID[9:16]] = [sceneID]
elif not sceneID in l8[sceneID[9:16]]:
l8[sceneID[9:16]].append(sceneID)
if args.allinpath:
sc = scenesearch(scenedata, sceneID, pathrowdict)
if len(sc) > 0:
for s in sc:
if not s in l8[sceneID[9:16]]:
print(
'Also adding scene {} to the processing list.'
.format(sceneID))
l8[sceneID[9:16]].append(s)
except Exception as e:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
print(exc_type, fname, exc_tb.tb_lineno)
print('ERROR: {} {} {} {}.'.format(sceneID, exc_type, fname,
exc_tb.tb_lineno))
ieo.logerror(sceneID, '{} {} {}'.format(exc_type, fname,
exc_tb.tb_lineno))
return l8, l47, cctype
if args.outdir:
outdir = os.path.join(args.outdir, os.path.basename(d))
if not os.path.isdir(outdir):
os.mkdir(outdir)
else:
outdir = d
if args.vrt:
if not d.endswith('vrt') and not d in [args.pixelqadir, args.fmaskdir]:
d = os.path.join(d, 'vrt')
flist = glob.glob(os.path.join(d, 'L*.vrt'))
else:
flist = glob.glob(os.path.join(d, 'L*.dat'))
if len(flist) > 0:
print('Now converting {} scenes to tiles from: \nCreating tiles in: {}'.format(len(flist), d, outdir))
for f in flist:
print('Converting: {} ({}/{})'.format(os.path.basename(f), flist.index(f) + 1, len(flist)))
# if dn in [2, 3]:
# rastertype = rastertypes[dn][os.path.basename(f)[2:3]]
# else:
rastertype = rastertypes[dn]
if dn < 2:
pixelqa = False
else:
pixelqa = True
try:
ieo.converttotiles(f, outdir, rastertype, pixelqa = pixelqa, overwrite = args.overwrite, noupdate = args.noupdate)
except Exception as e:
ieo.logerror(f, e)
print('ERROR with file {}:\n{}'.format(f,e))
# This makes a progress bar. I did not originally write it, nor do I remember from where I found the code.
readsofar = blocknum * blocksize
if totalsize > 0:
percent = readsofar * 1e2 / totalsize
s = "\r%5.1f%% %*d / %d" % (
percent, len(str(totalsize)), readsofar, totalsize)
sys.stderr.write(s)
if readsofar >= totalsize: # near the end
sys.stderr.write("\n")
else: # total size is unknown
sys.stderr.write("read %d\n" % (readsofar,))
if args.MBR: # define MBR for scene queries
args.MBR = args.MBR.split(',')
if len(args.MBR) != 4:
ieo.logerror('--MBR', 'Total number of coordinates does not equal four.', errorfile = errorfile)
print('Error: Improper number of coordinates for --MBR set (must be four). Either remove this option (will use default values) or fix. Exiting.')
sys.exit()
else:
args.MBR = getMBR()
# This section borrowed from https://pcjericks.github.io/py-gdalogr-cookbook/projection.html
# Lat/ Lon WGS-84 to local projection transformation
source = osr.SpatialReference() # Lat/Lon WGS-64
source.ImportFromEPSG(4326)
#target = osr.SpatialReference()
#i = ieo.prjstr.find(':') + 1
#target.ImportFromEPSG(int(ieo.prjstr[i:])) # EPSG code set in ieo.ini
target = ieo.prj
def populatelists(l8, l47, scenedata, localscenelist):
for sceneID in scenedata.keys():
acqDate = scenedata[sceneID]['acqDate']
path = scenedata[sceneID]['Path']
row = scenedata[sceneID]['Row']
scenesensor = scenedata[sceneID]['Sensor']
if args.ccland:
cc = scenedata[sceneID]['CCLand']
maxcc = args.maxccland
else:
cc = scenedata[sceneID]['CCFull']
maxcc = args.maxcc
sunEl = scenedata[sceneID]['sunEl']
SR = scenedata[sceneID]['SR_path']
proclevel = scenedata[sceneID]['proclevel']
try:
if (
not sceneID[:16] in localscenelist or args.ignorelocal
) and cc <= maxcc and sunEl >= args.minsunel and proclevel in proclevels: # Only run this for scenes that aren't present on disk or if we choose to ignore local copies.
# if (feature.GetField("SR_path") == None or args.ignorelocal) and feature.GetField("CCFull") <= args.maxcc and feature.GetField("sunEl") >= args.minsunel:
# sceneID = feature.GetField("sceneID")
if args.landsat:
if args.landsat != int(sceneID[2:3]):
continue
if args.path:
if args.path != path:
continue
# else:
# print(path)
if args.row:
if args.row != row:
continue
# else:
# print(row)
if args.sensor:
if sensor != scenesensor:
continue
year = int(sceneID[9:13])
doy = int(sceneID[13:16])
if args.startyear or args.endyear:
if args.startyear > args.endyear:
endyear = args.startyear
startyear = args.endyear
else:
endyear = args.endyear
startyear = args.startyear
if year < startyear or year > endyear:
continue
if args.startdoy and args.enddoy: # This might be programmed later to restrict dates to specific dates/ times of year
if args.startdoy < args.enddoy:
if doy < args.startdoy or doy > args.enddoy:
continue
else:
if args.startyear:
if year == startyear and doy < args.startdoy:
continue
if args.endyear:
if year == endyear and doy > args.enddoy:
continue
if doy > endday and doy < startday:
continue
if acqDate >= args.startdate and acqDate <= args.enddate:
#
print(
'Scene {}, cloud cover of {} percent, added to list.'.
format(sceneID, cc))
if not sceneID[9:16] in L7exclude and not sceneID[
2:
3] == '8': #(scenesensor == 'LANDSAT_TM' or scenesensor == 'LANDSAT_ETM' or 'LANDSAT_ETM_SLC_OFF') and
if not sceneID[9:16] in l47.keys():
l47[sceneID[9:16]] = [sceneID]
elif not sceneID in l47[sceneID[9:16]]:
l47[sceneID[9:16]].append(sceneID)
if args.allinpath:
sc = scenesearch(scenedata, sceneID, pathrowdict)
if len(sc) > 0:
for s in sc:
if not s in l47[sceneID[9:16]]:
print(
'Also adding scene {} to the processing list.'
.format(sceneID))
l47[sceneID[9:16]].append(s)
# elif scenesensor=='LANDSAT_ETM':
# l7.append(sceneID)
# elif scenesensor=='LANDSAT_ETM_SLC_OFF' and not sceneID[9:16] in L7exclude:
# l7slcoff.append(sceneID)
elif sceneID[2:3] == '8' and not sceneID[9:16] in L8exclude:
if not sceneID[9:16] in l8.keys():
l8[sceneID[9:16]] = [sceneID]
elif not sceneID in l8[sceneID[9:16]]:
l8[sceneID[9:16]].append(sceneID)
if args.allinpath:
sc = scenesearch(scenedata, sceneID, pathrowdict)
if len(sc) > 0:
for s in sc:
if not s in l8[sceneID[9:16]]:
print(
'Also adding scene {} to the processing list.'
.format(sceneID))
l8[sceneID[9:16]].append(s)
except Exception as e:
print('Error: {}'.format(e))
ieo.logerror(sceneID, e)
return l8, l47