def process_mean(cls, filenames, fout):
""" Calculates mean of all filenames, and per pixel variances """
start = datetime.datetime.now()
if len(filenames) > 0:
img = gippy.GeoImage(filenames)
imgout = gippy.GeoImage(fout, img, gippy.GDT_Float32, 2)
imgout.SetNoData(-32768)
img.Mean(imgout[0])
meanimg = imgout[0].Read()
for band in range(0, img.NumBands()):
data = img[band].Read()
mask = img[band].DataMask()
var = numpy.multiply(numpy.power(data - meanimg, 2), mask)
if band == 0:
totalvar = var
counts = mask
else:
totalvar = totalvar + var
counts = counts + mask
inds = numpy.where(counts == 0)
totalvar[inds] = -32768
inds = numpy.where(counts != 0)
totalvar[inds] = numpy.divide(totalvar[inds], counts[inds])
imgout[1].Write(totalvar)
t = datetime.datetime.now() - start
utils.verbose_out('%s: mean/var for %s files processed in %s' %
(os.path.basename(fout), len(filenames), t))
else:
raise Exception('No filenames provided')
return imgout
def process(self, products=None, overwrite=False, **kwargs):
"""Produce data products and save them to files.
Only one product; it's processed in the usual way, but for this
driver, it's extracted from the gzip file and saved (not
symlink/vsi). Method signature is largely for campatibilty with
the rest of gips, eg kwargs is unused.
"""
needed_products = self.needed_products(products, overwrite)
if len(needed_products) == 0:
utils.verbose_out('No new processing required.')
return
# sanity check that requested product & asset look ok
assert (needed_products.requested == {
_product_type: [_product_type]
} and _asset_type in self.assets)
asset = self.assets[_asset_type]
err_msg = 'Error creating product {} from {}'.format(
_product_type, os.path.basename(asset.filename))
with utils.error_handler(err_msg, continuable=True):
temp_fp = self.temp_product_filename(_sensor, _product_type)
# make gdal/gippy-readable path to the inner file
vsi_inner_path = '/vsigzip/' + asset.filename
os.symlink(vsi_inner_path, temp_fp)
archive_fp = self.archive_temp_path(temp_fp)
self.AddFile(_sensor, _product_type, archive_fp)
def fetch(cls, asset, tile, date):
"""Fetch to the stage.
Returns a list with one item, the full path to the staged asset.
"""
utils.verbose_out('{}: fetch tile {} for {}'.format(asset, tile, date),
3)
fn_prefix = cls._assets[asset]['fn-prefix']
stage_dir = cls.Repository.path('stage')
with utils.error_handler("Error downloading from " + cls._host, continuable=True), \
utils.make_temp_dir(prefix='fetchtmp', dir=stage_dir) as td_name:
qs_rv = cls.query_service(asset, tile, date)
if qs_rv is None:
return []
remote_fn = qs_rv['basename']
local_fn = fn_prefix + remote_fn
temp_fp = os.path.join(td_name, local_fn)
stage_fp = os.path.join(stage_dir, local_fn)
utils.verbose_out(
"Downloading {}, local name {}".format(remote_fn, local_fn), 2)
conn = cls.ftp_connect(asset, date)
with open(temp_fp, "wb") as temp_fo:
conn.retrbinary('RETR ' + remote_fn, temp_fo.write)
conn.quit()
os.rename(temp_fp, stage_fp)
return [stage_fp]
return []
def write_reduced(self, prod, fun, fout, meta, units):
""" apply a function to reduce to a daily value """
assetname = self._products[prod]['assets'][0]
layername = self._products[prod]['layers'][0]
bandnames = self._products[prod]['bands']
assetfile = self.assets[assetname].filename
ncroot = Dataset(assetfile)
var = ncroot.variables[layername]
missing = float(var.missing_value)
scale = var.scale_factor
assert scale == 1.0, "Handle non-unity scale functions"
hourly = np.ma.MaskedArray(var[:])
hourly.mask = (hourly == missing)
nb, ny, nx = hourly.shape
# apply reduce rule
daily = fun(hourly)
daily[daily.mask] = missing
utils.verbose_out('writing %s' % fout, 4)
imgout = gippy.GeoImage(fout, nx, ny, 1, gippy.GDT_Float32)
imgout[0].Write(np.array(np.flipud(daily)).astype('float32'))
imgout.SetBandName(prod, 1)
imgout.SetUnits(units)
imgout.SetNoData(missing)
imgout.SetProjection(self._projection)
imgout.SetAffine(np.array(self._geotransform))
imgout.SetMeta(self.prep_meta(assetfile, meta))
def ftp_connect(cls, asset, date):
"""Connect to an FTP server and chdir according to the args.
Returns the ftplib connection object."""
utils.verbose_out(
'Connecting to {}'.format(cls._assets[asset]['host']), 5)
if asset == 'IMERG-DAY-FINAL':
conn = ftplib.FTP(cls._assets[asset]['host'])
conn.login('*****@*****.**', '*****@*****.**')
conn.set_pasv(True)
working_directory = os.path.join(cls._assets[asset]['path'],
date.strftime('%Y'),
date.strftime('%m'),
date.strftime('%d'), 'gis')
elif asset == 'IMERG-DAY-LATE' or asset == 'IMERG-DAY-EARLY' or asset == 'IMERG-DAY-MID':
conn = ftplib.FTP(cls._assets[asset]['host'])
conn.login('*****@*****.**', '*****@*****.**')
conn.set_pasv(True)
working_directory = os.path.join(cls._assets[asset]['path'],
date.strftime('%Y'),
date.strftime('%m'))
elif asset == '3B42-DAY-LATE':
conn = ftplib.FTP(cls._assets[asset]['host'])
conn.login()
conn.set_pasv(True)
working_directory = os.path.join(cls._assets[asset]['path'],
date.strftime('%Y%m'))
utils.verbose_out('Changing to {}'.format(working_directory), 5)
conn.cwd(working_directory)
return conn
def query_provider(cls, asset, tile, date):
year, month, day = date.timetuple()[:3]
if asset != "ASM":
mainurl = "%s/%04d/%02d" % (cls._assets[asset]['url'], year, month)
pattern = cls._assets[asset]['re_pattern'] % (year, month, day)
else:
# asset ASM is for constants which all go into 1980-01-01
mainurl = cls._assets[asset]['url']
pattern = cls._assets[asset]['re_pattern'] % (0, 0, 0)
cpattern = re.compile(pattern)
with utils.error_handler("Error downloading"):
# obtain the list of files
response = cls.Repository.managed_request(mainurl, verbosity=2)
if response is None:
return None, None
for item in response.readlines():
# inspect the page and extract the full name of the needed file
if cpattern.search(item):
if 'xml' in item:
continue
basename = cpattern.findall(item)[0]
url = '/'.join([mainurl, basename])
return basename, url
utils.verbose_out(
"Unable to find a remote match for"
" {} at {}".format(pattern, mainurl), 4)
return None, None
def query_s3(cls, tile, date):
"""Look in S3 for modis asset components and assemble links to same."""
h, v = cls.parse_tile(tile)
prefix = 'MCD43A4.006/{}/{}/{}/'.format(h, v, date.strftime('%Y%j'))
keys = cls.s3_prefix_search(prefix)
tifs = []
qa_tifs = []
json_md = None
for k in keys:
if k.endswith('qa.TIF'):
qa_tifs.append(cls.s3_vsi_prefix(k))
elif k.endswith('.TIF'):
tifs.append(cls.s3_vsi_prefix(k))
elif k.endswith('_meta.json'):
json_md = cls._s3_url + k
if len(tifs) != len(qa_tifs) != 7 or json_md is None:
utils.verbose_out(
'Found no complete S3 asset for'
' (MCD43A4S3, {}, {})'.format(tile, date), 4)
return None
utils.verbose_out(
'Found complete S3 asset for'
' (MCD43A4S3, {}, {})'.format(tile, date), 3)
tifs.sort()
qa_tifs.sort()
# turn this: MCD43A4.006/12/04/2017330/MCD43A4.A2017330.h12v04.006.2017341210453_B01.TIF
# into this: MCD43A4.A2017330.h12v04.006.2017341210453_S3.json
asset_bn = os.path.basename(tifs[0])[:-8] + '_S3.json'
return {
'basename': asset_bn,
'json_md': json_md,
'tifs': tifs,
'qa_tifs': qa_tifs
}
def datafiles(self):
datafiles = super(prismAsset, self).datafiles()
datafiles = filter(lambda x: x.lower().endswith('.bil'), datafiles)
if len(datafiles) > 0:
indexfile = self.filename + '.index'
utils.verbose_out('indexfile: {}'.format(indexfile), 3)
List2File(datafiles, indexfile)
return datafiles
def check_hls_version(cls):
"""Once per runtime, confirm 1.4 is still usable."""
r = requests.head(_url_base + '/')
if r.status_code == 200:
verbose_out('HLS URL base `{}` confirmed valid'.format(_url_base), 5)
else:
raise requests.HTTPError('HLS URL base `{}` returned status code'
' {}; HLS version may have changed'.format(
_url_base, r.status_code))
def main():
title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF
# argument parsing
parser0 = GIPSParser(description=title)
parser = parser0.add_inventory_parser()
group = parser.add_argument_group('additional inventory options')
group.add_argument('--md',
help='Show dates using MM-DD',
action='store_true',
default=False)
group.add_argument(
'--rectify',
help=
'Instead of displaying or fetching inventory, rectify the inventory '
'database by comparing it against the present state of the data repos.',
action='store_true',
default=False)
args = parser0.parse_args()
cls = utils.gips_script_setup(args.command, args.stop_on_error)
with utils.error_handler():
print(title)
if args.rectify:
if not orm.use_orm():
raise ValueError("--rectify can only be used if"
" GIPS_ORM = True.")
for k, v in vars(args).items():
# Let the user know not to expect other options to effect rectify
if v and k not in ('rectify', 'verbose', 'command'):
msg = "INFO: Option '--{}' is has no effect on --rectify."
utils.verbose_out(msg.format(k), 1)
print("Rectifying inventory DB with filesystem archive:")
print("Rectifying assets:")
dbinv.rectify_assets(cls.Asset)
print("Rectifying products:")
dbinv.rectify_products(cls)
return
extents = SpatialExtent.factory(cls,
site=args.site,
rastermask=args.rastermask,
key=args.key,
where=args.where,
tiles=args.tiles,
pcov=args.pcov,
ptile=args.ptile)
for extent in extents:
inv = DataInventory(cls, extent,
TemporalExtent(args.dates, args.days),
**vars(args))
inv.pprint(md=args.md, size=args.size)
utils.gips_exit(
) # produce a summary error report then quit with a proper exit status
def __init__(self, bandnum, wvlen1, wvlen2, dtime, lat, lon, profile=False):
self.lat = lat
self.lon = lon
self.datetime = dtime
seconds = (dtime.second + dtime.microsecond / 1000000.) / 3600.
self.dtime = self.datetime.hour + self.datetime.minute / 60.0 + seconds
self.julianday = (dtime - datetime.datetime(dtime.year, 1, 1)).days + 1
self.model = atmospheric_model(self.julianday, lat)
#fout = open('atm.txt','w')
#fout.write('{:>5}{:>20}{:>20}\n'.format('Band','%T','Radiance'))
tmpdir = tempfile.mkdtemp()
pwd = os.getcwd()
os.chdir(tmpdir)
# Create link to MODTRAN data dir
if not os.path.lexists('DATA'):
os.symlink(self._datadir, 'DATA')
if profile:
mprofile = merraData.profile(lon, lat, dtime)
pressure = mprofile['pressure']
temp = mprofile['temp']
humidity = mprofile['humidity']
ozone = mprofile['ozone']
self.atmprofile = []
for i in range(0, len(pressure)):
c2c1 = self.card2c1(P=pressure[i], T=temp[i], H2O=humidity[i], O3=ozone[i])
self.atmprofile.append(c2c1)
else:
self.atmprofile = None
# Generate MODTRAN input files
# Determine if radiance or transmittance mode
rootnames = self.addband(bandnum, wvlen1, wvlen2)
List2File(rootnames, 'mod5root.in')
try:
# run output and get results
modout = commands.getstatusoutput('modtran')
#VerboseOut("MODTRAN Output:", 4)
#[VerboseOut(m, 4) for m in modout]
self.output = self.readoutput(bandnum)
verbose_out('MODTRAN Output: %s' % ' '.join([str(s) for s in self.output]), 4)
except:
verbose_out(modout, 4)
raise
# Change back to original directory
os.chdir(pwd)
#print 'MODTRAN dir: ', tmpdir
# Remove directory
shutil.rmtree(tmpdir)
def process(self, *args, **kwargs):
""" Process all requested products for this tile """
products = super(sarannualData, self).process(*args, **kwargs)
if len(products) == 0:
return
self.basename = self.basename + '_' + self.sensor_set[0]
for key, val in products.requested.items():
fname = os.path.join(self.path, self.basename + '_' + key)
# Verify that asset exists
a_type = self._products[val[0]]['assets'][0]
a_obj = self.assets.get(a_type)
if a_obj is None:
utils.verbose_out(
"Asset {} doesn't exist for tile {}".format(
a_type, self.id), 3)
continue
datafiles = None
with utils.error_handler(
"Error extracting files from asset {}".format(
a_obj.filename),
continuable=True):
datafiles = a_obj.extract()
if datafiles is None:
continue
if val[0] == 'sign':
bands = [
datafiles[b] for b in ["sl_HH", "sl_HV"] if b in datafiles
]
if len(bands) > 0:
img = gippy.GeoImage(bands)
img.SetNoData(0)
mask = gippy.GeoImage(datafiles['mask'], False)
img.AddMask(mask[0] == 255)
imgout = gippy.GeoImage(fname, img, gippy.GDT_Float32)
imgout.SetNoData(-32768)
for b in range(0, imgout.NumBands()):
imgout.SetBandName(img[b].Description(), b + 1)
(img[b].pow(2).log10() * 10 - 83.0).Process(imgout[b])
fname = imgout.Filename()
img = None
imgout = None
[
RemoveFiles([f], ['.hdr', '.aux.xml'])
for k, f in datafiles.items() if k != 'hdr'
]
if val[0] == 'fnf':
if 'C' in datafiles:
# rename both files to product name
os.rename(datafiles['C'], fname)
os.rename(datafiles['C'] + '.hdr', fname + '.hdr')
img = gippy.GeoImage(fname)
img.SetNoData(0)
img = None
self.AddFile(self.sensor_set[0], key, fname)
def add_data_sources(self):
""" Adds data sources to parser """
subparser = self.add_subparsers(dest='command')
sources = data_sources()
if len(sources) == 0:
verbose_out("There are no available data sources!", 1, sys.stderr)
for src, desc in sources.items():
p = subparser.add_parser(src,
help=desc,
parents=self.parent_parsers)
utils.import_data_class(src).add_filter_args(p)
def main():
verbose_out(
"\n\n\n DEPRECATION WARNING\n"
" ----------------------\n"
" 'gips_project' has moved.\n"
" It is now called 'gips_export',\n"
" and 'gips_project' will be \n"
" removed in a future release.\n\n\n",
0,
stream=sys.stderr,
)
export.main()
def process(self, products=None, overwrite=False):
"""Generate hls products."""
products = self.needed_products(products, overwrite)
if len(products) == 0:
verbose_out('No new processing required.', 5)
return
# thus we never have to write an `if val[0] == 'ref':` block
# (for other drivers do getattr(...)(*pl[1:]) for args, eg 'toa')
[getattr(self, 'process_' + pl[0])()
for pl in products.groups()['Standard'].values()]
indices = products.groups()['Index']
if indices:
[self.process_indices(ao, indices) for ao in self.assets.values()]
def process(self, *args, **kwargs):
"""Produce requested products."""
products = super(gpmData, self).process(*args, **kwargs)
if len(products) == 0:
return
# example products.requested:
# {'temp8tn': ['temp8tn'], 'clouds': ['clouds'], . . . }
# key is only used once far below, and val is only used for val[0].
for key, val in products.requested.items():
start = datetime.datetime.now()
prod_type = val[0]
asset, missingassets, availassets, allsds = \
self.asset_check(prod_type)
if not availassets:
# some products aren't available for every day but this is trying every day
VerboseOut(
'There are no available assets (%s) on %s for tile %s' % (
str(missingassets),
str(self.date),
str(self.id),
), 5)
continue
sensor = self._products[prod_type]['sensor']
fname = self.temp_product_filename(
sensor, prod_type) # moved to archive at end of loop
img = gippy.GeoImage(allsds)
imgdata = img.Read()
imgout = gippy.GeoImage(fname, img.XSize(), img.YSize(), 1,
gippy.GDT_Float32)
del img
imgout.SetNoData(29999.0)
imgout.SetOffset(0.0)
imgout.SetGain(0.1)
imgout.SetBandName('PrecipitationRate', 1)
imgout.SetProjection(self._projection)
imgout.SetAffine(
np.array(self._products[prod_type]['_geotransform']))
imgout[0].Write(imgdata)
# add product to inventory
archive_fp = self.archive_temp_path(fname)
self.AddFile(sensor, key, archive_fp)
del imgout # to cover for GDAL's internal problems
utils.verbose_out(' -> {}: processed in {}'.format(
os.path.basename(fname),
datetime.datetime.now() - start),
level=1)
def fetch(cls, asset, tile, date):
qs_rv = cls.query_service(asset, tile, date)
if qs_rv is None:
return []
basename, url = qs_rv['basename'], qs_rv['url']
with utils.error_handler("Asset fetch error ({})".format(url),
continuable=True):
response = cls.Repository.managed_request(url)
if response is None:
return []
outpath = os.path.join(cls.Repository.path('stage'), basename)
with open(outpath, 'wb') as fd:
fd.write(response.read())
utils.verbose_out('Retrieved ' + basename, 2)
return [outpath]
return []
def rectify_asset(f_name): # work function for _chunky_transaction()
a = asset_class(f_name)
(asset, created) = mao.update_or_create(asset=a.asset,
sensor=a.sensor,
tile=a.tile,
date=a.date,
name=f_name,
driver=driver)
asset.save()
touched_rows.add(asset.pk)
if created:
counts['add'] += 1
verbose_out("Asset added to database: " + f_name, 5)
else:
counts['update'] += 1
verbose_out("Asset found in database: " + f_name, 5)
def query_provider(cls, asset, tile, date):
"""Find out from the modis servers what assets are available.
Uses the given (asset, tile, date) tuple as a search key, and
returns a tuple: base-filename, url
"""
year, month, day = date.timetuple()[:3]
if asset == "MCD12Q1" and (month, day) != (1, 1):
utils.verbose_out(
"Cannot fetch MCD12Q1: Land cover data"
" are only available for Jan. 1",
1,
stream=sys.stderr)
return None, None
mainurl = "%s/%s.%02d.%02d" % (cls._assets[asset]['url'], str(year),
month, day)
pattern = '(%s.A%s%s.%s.\d{3}.\d{13}.hdf)' % (
asset, str(year), str(date.timetuple()[7]).zfill(3), tile)
cpattern = re.compile(pattern)
if datetime.datetime.today().date().weekday() == 2:
err_msg = ("Error downloading on a Wednesday;"
" possible planned MODIS provider downtime: " + mainurl)
else:
err_msg = "Error downloading: " + mainurl
with utils.error_handler(err_msg):
response = cls.Repository.managed_request(mainurl, verbosity=2)
if response is None:
return None, None
for item in response.readlines():
# screen-scrape the content of the page and extract the full name of the needed file
# (this step is needed because part of the filename, the creation timestamp, is
# effectively random).
if cpattern.search(item):
if 'xml' in item:
continue
basename = cpattern.findall(item)[0]
url = ''.join([mainurl, '/', basename])
return basename, url
utils.verbose_out(
'Unable to find remote match for '
'{} at {}'.format(pattern, mainurl), 4)
return None, None
def main():
title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF
parser0 = GIPSParser(datasources=False, description=title)
parser0.add_projdir_parser()
group = parser0.add_argument_group('masking options')
args = parser0.parse_args()
utils.gips_script_setup(stop_on_error=args.stop_on_error)
print title
# TODO - check that at least 1 of filemask or pmask is supplied
header = ['date', 'band', 'min', 'max', 'mean', 'sd', 'skew', 'count']
with utils.error_handler():
for projdir in args.projdir:
VerboseOut('Stats for Project directory: %s' % projdir, 1)
inv = ProjectInventory(projdir, args.products)
p_dates = {} # map each product to its list of valid dates
for date in inv.dates:
for p in inv.products(date):
p_dates.setdefault(p, []).append(date)
p_dates = {p: sorted(dl) for p, dl in p_dates.items()}
for p_type, valid_dates in p_dates.items():
stats_fn = os.path.join(projdir, p_type + '_stats.txt')
with open(stats_fn, 'w') as stats_fo:
sf = getattr(utils.settings(), 'STATS_FORMAT', {})
writer = csv.writer(stats_fo, **sf)
writer.writerow(header)
# print date, band description, and stats
for date in valid_dates:
img = inv[date].open(p_type)
date_str = date.strftime('%Y-%j')
utils.verbose_out('Computing stats for {} {}'.format(
p_type, date_str), 2)
for b in img:
stats = [str(s) for s in b.Stats()]
writer.writerow(
[date_str, b.Description()] + stats)
img = None
utils.gips_exit() # produce a summary error report then quit with a proper exit status
def initialize_composites(cls):
'''
This method is incomplete, but will ensure that the composites
directory is boostrapped for running process composites.
INCOMPLETE at this time.
'''
lta_tif = os.path.join(cls.Asset.Repository.path('composites'),
'lta.tif')
ltadpat = os.path.join(cls.Asset.Repository.path('composites'), 'ltad',
'ltad???.tif')
ltad_tifs = glob.glob(ltadpat)
if os.path.exists(lta_tif) and len(ltad_tifs) == 366:
utils.verbose_out('lta composites already initialized', 2)
return
a = inv[inv.dates[0]].tiles[cls.Asset.Repository._the_tile].open('aod')
a[0] = a[0] * 0 + a[0].NoDataValue()
a.Process(ltatif)
def query_provider(cls, asset, tile, date):
"""Search for a matching asset in the CHIRPS ftp store.
Returns (basename, None) on success; (None, None) otherwise."""
conn = cls.ftp_connect(asset, date)
filenames = [
fn for fn in conn.nlst() if date.strftime('%Y.%m.%d') in fn
]
conn.quit()
f_cnt = len(filenames)
if f_cnt == 0:
return None, None
if f_cnt > 1:
utils.verbose_out("Too many assets found: " + repr(filenames), 1,
sys.stderr)
raise ValueError("Can't decide between {} assets".format(f_cnt),
filenames)
return filenames[
0], None # URL is pointless for ftp, have to chdir manually anyway
def fetch(cls, asset, tile, date):
"""Fetch the AOD asset matching the ATD."""
query_rv = cls.query_service(asset, tile, date)
if query_rv is None:
return []
bn, url = query_rv['basename'], query_rv['url']
stage_dn = cls.Repository.path('stage')
stage_fp = os.path.join(stage_dn, bn)
with utils.make_temp_dir(prefix='fetch', dir=stage_dn) as tmp_dn:
tmp_fp = os.path.join(tmp_dn, bn)
utils.verbose_out(
'Fetching from {}, saving in {}'.format(url, tmp_fp), 5)
resp = requests.get(url, stream=True)
resp.raise_for_status()
with open(tmp_fp, 'w') as tmp_fo:
for chunk in resp.iter_content(chunk_size=(1024**2)):
tmp_fo.write(chunk)
os.rename(tmp_fp, stage_fp)
return [stage_fp]
def query_provider(cls, asset, tile, date):
"""Find out from the NASA servers what AOD assets are available."""
if asset not in cls._assets:
raise ValueError('{} has no defined asset for {}'.format(
cls.Repository.name, asset))
with utils.error_handler("Error querying " + cls._host,
continuable=True):
# example full json query URL:
# https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD08_D3/2017/145.json
url_head = cls._assets[asset]['url'] + date.strftime('%Y/%j')
query_url = url_head + '.json'
utils.verbose_out('Downloading ' + query_url, 5)
resp = requests.get(query_url)
resp.raise_for_status() # some errors don't raise otherwise
found_assets = resp.json()
if len(found_assets) != 1:
raise ValueError('Expected one {} asset, found {}'.format(
asset, len(resp)))
filename = str(found_assets[0]['name']) # str() to prevent unicode
return filename, url_head + '/' + filename
return None, None
def rectify_product(full_fn):
bfn_parts = basename(full_fn).split('_')
if not len(bfn_parts) == 4:
_match_failure_report(
full_fn,
"Failure to parse: Wrong number of '_'-delimited substrings.")
return
# extract metadata about the file
(tile, date_str, sensor, product) = bfn_parts
date_pattern = data_class.Asset.Repository._datedir
try:
date = datetime.datetime.strptime(date_str, date_pattern).date()
except Exception:
verbose_out(traceback.format_exc(), 4, sys.stderr)
msg = "Failure to parse date: '{}' didn't adhere to pattern '{}'."
_match_failure_report(full_fn, msg.format(date_str, date_pattern))
return
(product, created) = mpo.update_or_create(product=product,
sensor=sensor,
tile=tile,
date=date,
driver=driver,
name=full_fn)
product.save()
# TODO can subtract this item from starting_keys each time and possibly save some memory and time
touched_rows.add(product.pk)
if created:
counts['add'] += 1
verbose_out("Product added to database: " + full_fn, 5)
else:
counts['update'] += 1
verbose_out("Product found in database: " + full_fn, 5)
def fetch(cls, asset, tile, date):
# The nassgeodata site is known to have an invalid certificate.
# We don't want to clutter up the output with SSL warnings.
if asset == _cdlmkii:
verbose_out("Fetching not supported for cdlmkii", 2)
return []
verbose_out("Fetching tile for {} on {}".format(tile, date.year), 2)
query_rv = cls.query_service(asset, tile, date)
if query_rv is None:
verbose_out("No CDL data for {} on {}".format(tile, date.year), 2)
return []
file_response = requests.get(
query_rv['url'], verify=False, stream=True)
with utils.make_temp_dir(
prefix='fetch', dir=cls.Repository.path('stage')) as tmp_dir:
fname = "{}_{}_cdl_cdl.tif".format(tile, date.year)
tmp_fname = tmp_dir + '/' + fname
with open(tmp_fname, 'w') as asset:
asset.write(file_response.content)
imgout = GeoImage(tmp_fname, True)
imgout.SetNoData(0)
imgout.SetMeta('GIPS_Version', gips.__version__)
imgout = None
shutil.copy(tmp_fname, cls.Repository.path('stage'))
def fetch(cls, asset, tile, date):
"""Standard Asset.fetch implementation for downloading assets."""
asset_info = cls.query_service(asset, tile, date)
if asset_info is None:
return []
basename, url = asset_info['basename'], asset_info['url']
basename = asset_info['basename']
url = asset_info['url']
outpath = os.path.join(cls.Repository.path('stage'), basename)
with utils.error_handler("Error fetching {} from {}".format(
basename, url),
continuable=True):
# obtain the data
response = cls.Repository.managed_request(url)
if response is None:
return []
stage_dir = cls.Repository.path('stage')
with utils.make_temp_dir(prefix='fetch', dir=stage_dir) as tmp_dn:
tmp_outpath = os.path.join(tmp_dn, basename)
with open(tmp_outpath, 'w') as fd:
fd.write(response.read())
# verify that it is a netcdf file
try:
ncroot = Dataset(tmp_outpath)
os.rename(tmp_outpath, outpath)
except Exception as e:
text = ''
if e.message.endswith('Unknown file format'):
token = 'Authorize NASA GESDISC DATA ARCHIVE'
html = open(tmp_outpath, 'r').read(100000)
if token in html:
text = ('\n\nYou need to {t} \nfor your NASA '
'EarthData login.\n').format(t=token)
raise Exception(e.message + text)
utils.verbose_out('Retrieved ' + basename, 2)
return [outpath]
def fetch(cls, asset, tile, date):
utils.verbose_out('%s: fetch tile %s for %s' % (asset, tile, date), 3)
qs_rv = cls.query_service(asset, tile, date)
if qs_rv is None:
return []
asset_fn = qs_rv['basename']
with utils.error_handler("Error downloading from " +
cls._assets[asset]['host'],
continuable=True):
ftp = cls.ftp_connect(
asset, date) # starts chdir'd to the right directory
stage_dir_fp = cls.Repository.path('stage')
stage_fp = os.path.join(stage_dir_fp, asset_fn)
with utils.make_temp_dir(prefix='fetchtmp',
dir=stage_dir_fp) as td_name:
temp_fp = os.path.join(td_name, asset_fn)
utils.verbose_out("Downloading " + asset_fn, 2)
with open(temp_fp, "wb") as temp_fo:
ftp.retrbinary('RETR ' + asset_fn, temp_fo.write)
ftp.quit()
os.rename(temp_fp, stage_fp)
return [stage_fp]
return []
def process_indices(self, a_obj, indices):
"""Make the given indices from the given asset.
indices' values are the keys, split by hyphen, eg {ndvi-toa':
['ndvi', 'toa']} (technically it's useless here, as no argumentated
products are supported; doing it for consistency).
"""
# TODO another place that can be DRY'd out; several drivers process similarly
verbose_out('Starting processing on {} for {}'.format(
a_obj.atd_str(), indices.keys()), 4)
gippy_input = {} # gippy wants p-types mapped to output filenames
tempfps_to_ptypes = {} # AddFile needs a map of p-types to filenames
for prod_type, pt_split in indices.items():
temp_fp = self.temp_product_filename(a_obj.sensor, prod_type)
gippy_input[pt_split[0]] = temp_fp
tempfps_to_ptypes[temp_fp] = prod_type
prodout = gippy.algorithms.Indices(
a_obj.load_image(), gippy_input, self.prep_meta(a_obj.filename))
for temp_fp in prodout.values():
archived_fp = self.archive_temp_path(temp_fp)
self.AddFile(a_obj.sensor, tempfps_to_ptypes[temp_fp], archived_fp)
def _calculate_estimate(filename):
val, var = cls._read_point(filename, roi, nodata)
aod = numpy.nan
norm = numpy.nan
# Negative values don't make sense
if val < 0:
val = 0
if var == 0:
# There is only one observation, so make up
# the variance.
if val == 0:
var = 0.15
else:
var = val / 2
if not numpy.isnan(val) and not numpy.isnan(var):
aod = val / var
norm = 1.0 / var
utils.verbose_out('AOD: LTA-Daily = %s, %s' % (val, var),
3)
return aod, norm
