def process_mean(cls, filenames, fout): """ Calculates mean of all filenames, and per pixel variances """ start = datetime.datetime.now() if len(filenames) > 0: img = gippy.GeoImage(filenames) imgout = gippy.GeoImage(fout, img, gippy.GDT_Float32, 2) imgout.SetNoData(-32768) img.Mean(imgout[0]) meanimg = imgout[0].Read() for band in range(0, img.NumBands()): data = img[band].Read() mask = img[band].DataMask() var = numpy.multiply(numpy.power(data - meanimg, 2), mask) if band == 0: totalvar = var counts = mask else: totalvar = totalvar + var counts = counts + mask inds = numpy.where(counts == 0) totalvar[inds] = -32768 inds = numpy.where(counts != 0) totalvar[inds] = numpy.divide(totalvar[inds], counts[inds]) imgout[1].Write(totalvar) t = datetime.datetime.now() - start utils.verbose_out('%s: mean/var for %s files processed in %s' % (os.path.basename(fout), len(filenames), t)) else: raise Exception('No filenames provided') return imgout
def process(self, products=None, overwrite=False, **kwargs): """Produce data products and save them to files. Only one product; it's processed in the usual way, but for this driver, it's extracted from the gzip file and saved (not symlink/vsi). Method signature is largely for campatibilty with the rest of gips, eg kwargs is unused. """ needed_products = self.needed_products(products, overwrite) if len(needed_products) == 0: utils.verbose_out('No new processing required.') return # sanity check that requested product & asset look ok assert (needed_products.requested == { _product_type: [_product_type] } and _asset_type in self.assets) asset = self.assets[_asset_type] err_msg = 'Error creating product {} from {}'.format( _product_type, os.path.basename(asset.filename)) with utils.error_handler(err_msg, continuable=True): temp_fp = self.temp_product_filename(_sensor, _product_type) # make gdal/gippy-readable path to the inner file vsi_inner_path = '/vsigzip/' + asset.filename os.symlink(vsi_inner_path, temp_fp) archive_fp = self.archive_temp_path(temp_fp) self.AddFile(_sensor, _product_type, archive_fp)
def fetch(cls, asset, tile, date): """Fetch to the stage. Returns a list with one item, the full path to the staged asset. """ utils.verbose_out('{}: fetch tile {} for {}'.format(asset, tile, date), 3) fn_prefix = cls._assets[asset]['fn-prefix'] stage_dir = cls.Repository.path('stage') with utils.error_handler("Error downloading from " + cls._host, continuable=True), \ utils.make_temp_dir(prefix='fetchtmp', dir=stage_dir) as td_name: qs_rv = cls.query_service(asset, tile, date) if qs_rv is None: return [] remote_fn = qs_rv['basename'] local_fn = fn_prefix + remote_fn temp_fp = os.path.join(td_name, local_fn) stage_fp = os.path.join(stage_dir, local_fn) utils.verbose_out( "Downloading {}, local name {}".format(remote_fn, local_fn), 2) conn = cls.ftp_connect(asset, date) with open(temp_fp, "wb") as temp_fo: conn.retrbinary('RETR ' + remote_fn, temp_fo.write) conn.quit() os.rename(temp_fp, stage_fp) return [stage_fp] return []
def write_reduced(self, prod, fun, fout, meta, units): """ apply a function to reduce to a daily value """ assetname = self._products[prod]['assets'][0] layername = self._products[prod]['layers'][0] bandnames = self._products[prod]['bands'] assetfile = self.assets[assetname].filename ncroot = Dataset(assetfile) var = ncroot.variables[layername] missing = float(var.missing_value) scale = var.scale_factor assert scale == 1.0, "Handle non-unity scale functions" hourly = np.ma.MaskedArray(var[:]) hourly.mask = (hourly == missing) nb, ny, nx = hourly.shape # apply reduce rule daily = fun(hourly) daily[daily.mask] = missing utils.verbose_out('writing %s' % fout, 4) imgout = gippy.GeoImage(fout, nx, ny, 1, gippy.GDT_Float32) imgout[0].Write(np.array(np.flipud(daily)).astype('float32')) imgout.SetBandName(prod, 1) imgout.SetUnits(units) imgout.SetNoData(missing) imgout.SetProjection(self._projection) imgout.SetAffine(np.array(self._geotransform)) imgout.SetMeta(self.prep_meta(assetfile, meta))
def ftp_connect(cls, asset, date): """Connect to an FTP server and chdir according to the args. Returns the ftplib connection object.""" utils.verbose_out( 'Connecting to {}'.format(cls._assets[asset]['host']), 5) if asset == 'IMERG-DAY-FINAL': conn = ftplib.FTP(cls._assets[asset]['host']) conn.login('*****@*****.**', '*****@*****.**') conn.set_pasv(True) working_directory = os.path.join(cls._assets[asset]['path'], date.strftime('%Y'), date.strftime('%m'), date.strftime('%d'), 'gis') elif asset == 'IMERG-DAY-LATE' or asset == 'IMERG-DAY-EARLY' or asset == 'IMERG-DAY-MID': conn = ftplib.FTP(cls._assets[asset]['host']) conn.login('*****@*****.**', '*****@*****.**') conn.set_pasv(True) working_directory = os.path.join(cls._assets[asset]['path'], date.strftime('%Y'), date.strftime('%m')) elif asset == '3B42-DAY-LATE': conn = ftplib.FTP(cls._assets[asset]['host']) conn.login() conn.set_pasv(True) working_directory = os.path.join(cls._assets[asset]['path'], date.strftime('%Y%m')) utils.verbose_out('Changing to {}'.format(working_directory), 5) conn.cwd(working_directory) return conn
def query_provider(cls, asset, tile, date): year, month, day = date.timetuple()[:3] if asset != "ASM": mainurl = "%s/%04d/%02d" % (cls._assets[asset]['url'], year, month) pattern = cls._assets[asset]['re_pattern'] % (year, month, day) else: # asset ASM is for constants which all go into 1980-01-01 mainurl = cls._assets[asset]['url'] pattern = cls._assets[asset]['re_pattern'] % (0, 0, 0) cpattern = re.compile(pattern) with utils.error_handler("Error downloading"): # obtain the list of files response = cls.Repository.managed_request(mainurl, verbosity=2) if response is None: return None, None for item in response.readlines(): # inspect the page and extract the full name of the needed file if cpattern.search(item): if 'xml' in item: continue basename = cpattern.findall(item)[0] url = '/'.join([mainurl, basename]) return basename, url utils.verbose_out( "Unable to find a remote match for" " {} at {}".format(pattern, mainurl), 4) return None, None
def query_s3(cls, tile, date): """Look in S3 for modis asset components and assemble links to same.""" h, v = cls.parse_tile(tile) prefix = 'MCD43A4.006/{}/{}/{}/'.format(h, v, date.strftime('%Y%j')) keys = cls.s3_prefix_search(prefix) tifs = [] qa_tifs = [] json_md = None for k in keys: if k.endswith('qa.TIF'): qa_tifs.append(cls.s3_vsi_prefix(k)) elif k.endswith('.TIF'): tifs.append(cls.s3_vsi_prefix(k)) elif k.endswith('_meta.json'): json_md = cls._s3_url + k if len(tifs) != len(qa_tifs) != 7 or json_md is None: utils.verbose_out( 'Found no complete S3 asset for' ' (MCD43A4S3, {}, {})'.format(tile, date), 4) return None utils.verbose_out( 'Found complete S3 asset for' ' (MCD43A4S3, {}, {})'.format(tile, date), 3) tifs.sort() qa_tifs.sort() # turn this: MCD43A4.006/12/04/2017330/MCD43A4.A2017330.h12v04.006.2017341210453_B01.TIF # into this: MCD43A4.A2017330.h12v04.006.2017341210453_S3.json asset_bn = os.path.basename(tifs[0])[:-8] + '_S3.json' return { 'basename': asset_bn, 'json_md': json_md, 'tifs': tifs, 'qa_tifs': qa_tifs }
def datafiles(self): datafiles = super(prismAsset, self).datafiles() datafiles = filter(lambda x: x.lower().endswith('.bil'), datafiles) if len(datafiles) > 0: indexfile = self.filename + '.index' utils.verbose_out('indexfile: {}'.format(indexfile), 3) List2File(datafiles, indexfile) return datafiles
def check_hls_version(cls): """Once per runtime, confirm 1.4 is still usable.""" r = requests.head(_url_base + '/') if r.status_code == 200: verbose_out('HLS URL base `{}` confirmed valid'.format(_url_base), 5) else: raise requests.HTTPError('HLS URL base `{}` returned status code' ' {}; HLS version may have changed'.format( _url_base, r.status_code))
def main(): title = Colors.BOLD + 'GIPS Data Inventory (v%s)' % gipsversion + Colors.OFF # argument parsing parser0 = GIPSParser(description=title) parser = parser0.add_inventory_parser() group = parser.add_argument_group('additional inventory options') group.add_argument('--md', help='Show dates using MM-DD', action='store_true', default=False) group.add_argument( '--rectify', help= 'Instead of displaying or fetching inventory, rectify the inventory ' 'database by comparing it against the present state of the data repos.', action='store_true', default=False) args = parser0.parse_args() cls = utils.gips_script_setup(args.command, args.stop_on_error) with utils.error_handler(): print(title) if args.rectify: if not orm.use_orm(): raise ValueError("--rectify can only be used if" " GIPS_ORM = True.") for k, v in vars(args).items(): # Let the user know not to expect other options to effect rectify if v and k not in ('rectify', 'verbose', 'command'): msg = "INFO: Option '--{}' is has no effect on --rectify." utils.verbose_out(msg.format(k), 1) print("Rectifying inventory DB with filesystem archive:") print("Rectifying assets:") dbinv.rectify_assets(cls.Asset) print("Rectifying products:") dbinv.rectify_products(cls) return extents = SpatialExtent.factory(cls, site=args.site, rastermask=args.rastermask, key=args.key, where=args.where, tiles=args.tiles, pcov=args.pcov, ptile=args.ptile) for extent in extents: inv = DataInventory(cls, extent, TemporalExtent(args.dates, args.days), **vars(args)) inv.pprint(md=args.md, size=args.size) utils.gips_exit( ) # produce a summary error report then quit with a proper exit status
def __init__(self, bandnum, wvlen1, wvlen2, dtime, lat, lon, profile=False): self.lat = lat self.lon = lon self.datetime = dtime seconds = (dtime.second + dtime.microsecond / 1000000.) / 3600. self.dtime = self.datetime.hour + self.datetime.minute / 60.0 + seconds self.julianday = (dtime - datetime.datetime(dtime.year, 1, 1)).days + 1 self.model = atmospheric_model(self.julianday, lat) #fout = open('atm.txt','w') #fout.write('{:>5}{:>20}{:>20}\n'.format('Band','%T','Radiance')) tmpdir = tempfile.mkdtemp() pwd = os.getcwd() os.chdir(tmpdir) # Create link to MODTRAN data dir if not os.path.lexists('DATA'): os.symlink(self._datadir, 'DATA') if profile: mprofile = merraData.profile(lon, lat, dtime) pressure = mprofile['pressure'] temp = mprofile['temp'] humidity = mprofile['humidity'] ozone = mprofile['ozone'] self.atmprofile = [] for i in range(0, len(pressure)): c2c1 = self.card2c1(P=pressure[i], T=temp[i], H2O=humidity[i], O3=ozone[i]) self.atmprofile.append(c2c1) else: self.atmprofile = None # Generate MODTRAN input files # Determine if radiance or transmittance mode rootnames = self.addband(bandnum, wvlen1, wvlen2) List2File(rootnames, 'mod5root.in') try: # run output and get results modout = commands.getstatusoutput('modtran') #VerboseOut("MODTRAN Output:", 4) #[VerboseOut(m, 4) for m in modout] self.output = self.readoutput(bandnum) verbose_out('MODTRAN Output: %s' % ' '.join([str(s) for s in self.output]), 4) except: verbose_out(modout, 4) raise # Change back to original directory os.chdir(pwd) #print 'MODTRAN dir: ', tmpdir # Remove directory shutil.rmtree(tmpdir)
def process(self, *args, **kwargs): """ Process all requested products for this tile """ products = super(sarannualData, self).process(*args, **kwargs) if len(products) == 0: return self.basename = self.basename + '_' + self.sensor_set[0] for key, val in products.requested.items(): fname = os.path.join(self.path, self.basename + '_' + key) # Verify that asset exists a_type = self._products[val[0]]['assets'][0] a_obj = self.assets.get(a_type) if a_obj is None: utils.verbose_out( "Asset {} doesn't exist for tile {}".format( a_type, self.id), 3) continue datafiles = None with utils.error_handler( "Error extracting files from asset {}".format( a_obj.filename), continuable=True): datafiles = a_obj.extract() if datafiles is None: continue if val[0] == 'sign': bands = [ datafiles[b] for b in ["sl_HH", "sl_HV"] if b in datafiles ] if len(bands) > 0: img = gippy.GeoImage(bands) img.SetNoData(0) mask = gippy.GeoImage(datafiles['mask'], False) img.AddMask(mask[0] == 255) imgout = gippy.GeoImage(fname, img, gippy.GDT_Float32) imgout.SetNoData(-32768) for b in range(0, imgout.NumBands()): imgout.SetBandName(img[b].Description(), b + 1) (img[b].pow(2).log10() * 10 - 83.0).Process(imgout[b]) fname = imgout.Filename() img = None imgout = None [ RemoveFiles([f], ['.hdr', '.aux.xml']) for k, f in datafiles.items() if k != 'hdr' ] if val[0] == 'fnf': if 'C' in datafiles: # rename both files to product name os.rename(datafiles['C'], fname) os.rename(datafiles['C'] + '.hdr', fname + '.hdr') img = gippy.GeoImage(fname) img.SetNoData(0) img = None self.AddFile(self.sensor_set[0], key, fname)
def add_data_sources(self): """ Adds data sources to parser """ subparser = self.add_subparsers(dest='command') sources = data_sources() if len(sources) == 0: verbose_out("There are no available data sources!", 1, sys.stderr) for src, desc in sources.items(): p = subparser.add_parser(src, help=desc, parents=self.parent_parsers) utils.import_data_class(src).add_filter_args(p)
def main(): verbose_out( "\n\n\n DEPRECATION WARNING\n" " ----------------------\n" " 'gips_project' has moved.\n" " It is now called 'gips_export',\n" " and 'gips_project' will be \n" " removed in a future release.\n\n\n", 0, stream=sys.stderr, ) export.main()
def process(self, products=None, overwrite=False): """Generate hls products.""" products = self.needed_products(products, overwrite) if len(products) == 0: verbose_out('No new processing required.', 5) return # thus we never have to write an `if val[0] == 'ref':` block # (for other drivers do getattr(...)(*pl[1:]) for args, eg 'toa') [getattr(self, 'process_' + pl[0])() for pl in products.groups()['Standard'].values()] indices = products.groups()['Index'] if indices: [self.process_indices(ao, indices) for ao in self.assets.values()]
def process(self, *args, **kwargs): """Produce requested products.""" products = super(gpmData, self).process(*args, **kwargs) if len(products) == 0: return # example products.requested: # {'temp8tn': ['temp8tn'], 'clouds': ['clouds'], . . . } # key is only used once far below, and val is only used for val[0]. for key, val in products.requested.items(): start = datetime.datetime.now() prod_type = val[0] asset, missingassets, availassets, allsds = \ self.asset_check(prod_type) if not availassets: # some products aren't available for every day but this is trying every day VerboseOut( 'There are no available assets (%s) on %s for tile %s' % ( str(missingassets), str(self.date), str(self.id), ), 5) continue sensor = self._products[prod_type]['sensor'] fname = self.temp_product_filename( sensor, prod_type) # moved to archive at end of loop img = gippy.GeoImage(allsds) imgdata = img.Read() imgout = gippy.GeoImage(fname, img.XSize(), img.YSize(), 1, gippy.GDT_Float32) del img imgout.SetNoData(29999.0) imgout.SetOffset(0.0) imgout.SetGain(0.1) imgout.SetBandName('PrecipitationRate', 1) imgout.SetProjection(self._projection) imgout.SetAffine( np.array(self._products[prod_type]['_geotransform'])) imgout[0].Write(imgdata) # add product to inventory archive_fp = self.archive_temp_path(fname) self.AddFile(sensor, key, archive_fp) del imgout # to cover for GDAL's internal problems utils.verbose_out(' -> {}: processed in {}'.format( os.path.basename(fname), datetime.datetime.now() - start), level=1)
def fetch(cls, asset, tile, date): qs_rv = cls.query_service(asset, tile, date) if qs_rv is None: return [] basename, url = qs_rv['basename'], qs_rv['url'] with utils.error_handler("Asset fetch error ({})".format(url), continuable=True): response = cls.Repository.managed_request(url) if response is None: return [] outpath = os.path.join(cls.Repository.path('stage'), basename) with open(outpath, 'wb') as fd: fd.write(response.read()) utils.verbose_out('Retrieved ' + basename, 2) return [outpath] return []
def rectify_asset(f_name): # work function for _chunky_transaction() a = asset_class(f_name) (asset, created) = mao.update_or_create(asset=a.asset, sensor=a.sensor, tile=a.tile, date=a.date, name=f_name, driver=driver) asset.save() touched_rows.add(asset.pk) if created: counts['add'] += 1 verbose_out("Asset added to database: " + f_name, 5) else: counts['update'] += 1 verbose_out("Asset found in database: " + f_name, 5)
def query_provider(cls, asset, tile, date): """Find out from the modis servers what assets are available. Uses the given (asset, tile, date) tuple as a search key, and returns a tuple: base-filename, url """ year, month, day = date.timetuple()[:3] if asset == "MCD12Q1" and (month, day) != (1, 1): utils.verbose_out( "Cannot fetch MCD12Q1: Land cover data" " are only available for Jan. 1", 1, stream=sys.stderr) return None, None mainurl = "%s/%s.%02d.%02d" % (cls._assets[asset]['url'], str(year), month, day) pattern = '(%s.A%s%s.%s.\d{3}.\d{13}.hdf)' % ( asset, str(year), str(date.timetuple()[7]).zfill(3), tile) cpattern = re.compile(pattern) if datetime.datetime.today().date().weekday() == 2: err_msg = ("Error downloading on a Wednesday;" " possible planned MODIS provider downtime: " + mainurl) else: err_msg = "Error downloading: " + mainurl with utils.error_handler(err_msg): response = cls.Repository.managed_request(mainurl, verbosity=2) if response is None: return None, None for item in response.readlines(): # screen-scrape the content of the page and extract the full name of the needed file # (this step is needed because part of the filename, the creation timestamp, is # effectively random). if cpattern.search(item): if 'xml' in item: continue basename = cpattern.findall(item)[0] url = ''.join([mainurl, '/', basename]) return basename, url utils.verbose_out( 'Unable to find remote match for ' '{} at {}'.format(pattern, mainurl), 4) return None, None
def main(): title = Colors.BOLD + 'GIPS Image Statistics (v%s)' % __version__ + Colors.OFF parser0 = GIPSParser(datasources=False, description=title) parser0.add_projdir_parser() group = parser0.add_argument_group('masking options') args = parser0.parse_args() utils.gips_script_setup(stop_on_error=args.stop_on_error) print title # TODO - check that at least 1 of filemask or pmask is supplied header = ['date', 'band', 'min', 'max', 'mean', 'sd', 'skew', 'count'] with utils.error_handler(): for projdir in args.projdir: VerboseOut('Stats for Project directory: %s' % projdir, 1) inv = ProjectInventory(projdir, args.products) p_dates = {} # map each product to its list of valid dates for date in inv.dates: for p in inv.products(date): p_dates.setdefault(p, []).append(date) p_dates = {p: sorted(dl) for p, dl in p_dates.items()} for p_type, valid_dates in p_dates.items(): stats_fn = os.path.join(projdir, p_type + '_stats.txt') with open(stats_fn, 'w') as stats_fo: sf = getattr(utils.settings(), 'STATS_FORMAT', {}) writer = csv.writer(stats_fo, **sf) writer.writerow(header) # print date, band description, and stats for date in valid_dates: img = inv[date].open(p_type) date_str = date.strftime('%Y-%j') utils.verbose_out('Computing stats for {} {}'.format( p_type, date_str), 2) for b in img: stats = [str(s) for s in b.Stats()] writer.writerow( [date_str, b.Description()] + stats) img = None utils.gips_exit() # produce a summary error report then quit with a proper exit status
def initialize_composites(cls): ''' This method is incomplete, but will ensure that the composites directory is boostrapped for running process composites. INCOMPLETE at this time. ''' lta_tif = os.path.join(cls.Asset.Repository.path('composites'), 'lta.tif') ltadpat = os.path.join(cls.Asset.Repository.path('composites'), 'ltad', 'ltad???.tif') ltad_tifs = glob.glob(ltadpat) if os.path.exists(lta_tif) and len(ltad_tifs) == 366: utils.verbose_out('lta composites already initialized', 2) return a = inv[inv.dates[0]].tiles[cls.Asset.Repository._the_tile].open('aod') a[0] = a[0] * 0 + a[0].NoDataValue() a.Process(ltatif)
def query_provider(cls, asset, tile, date): """Search for a matching asset in the CHIRPS ftp store. Returns (basename, None) on success; (None, None) otherwise.""" conn = cls.ftp_connect(asset, date) filenames = [ fn for fn in conn.nlst() if date.strftime('%Y.%m.%d') in fn ] conn.quit() f_cnt = len(filenames) if f_cnt == 0: return None, None if f_cnt > 1: utils.verbose_out("Too many assets found: " + repr(filenames), 1, sys.stderr) raise ValueError("Can't decide between {} assets".format(f_cnt), filenames) return filenames[ 0], None # URL is pointless for ftp, have to chdir manually anyway
def fetch(cls, asset, tile, date): """Fetch the AOD asset matching the ATD.""" query_rv = cls.query_service(asset, tile, date) if query_rv is None: return [] bn, url = query_rv['basename'], query_rv['url'] stage_dn = cls.Repository.path('stage') stage_fp = os.path.join(stage_dn, bn) with utils.make_temp_dir(prefix='fetch', dir=stage_dn) as tmp_dn: tmp_fp = os.path.join(tmp_dn, bn) utils.verbose_out( 'Fetching from {}, saving in {}'.format(url, tmp_fp), 5) resp = requests.get(url, stream=True) resp.raise_for_status() with open(tmp_fp, 'w') as tmp_fo: for chunk in resp.iter_content(chunk_size=(1024**2)): tmp_fo.write(chunk) os.rename(tmp_fp, stage_fp) return [stage_fp]
def query_provider(cls, asset, tile, date): """Find out from the NASA servers what AOD assets are available.""" if asset not in cls._assets: raise ValueError('{} has no defined asset for {}'.format( cls.Repository.name, asset)) with utils.error_handler("Error querying " + cls._host, continuable=True): # example full json query URL: # https://ladsweb.modaps.eosdis.nasa.gov/archive/allData/6/MOD08_D3/2017/145.json url_head = cls._assets[asset]['url'] + date.strftime('%Y/%j') query_url = url_head + '.json' utils.verbose_out('Downloading ' + query_url, 5) resp = requests.get(query_url) resp.raise_for_status() # some errors don't raise otherwise found_assets = resp.json() if len(found_assets) != 1: raise ValueError('Expected one {} asset, found {}'.format( asset, len(resp))) filename = str(found_assets[0]['name']) # str() to prevent unicode return filename, url_head + '/' + filename return None, None
def rectify_product(full_fn): bfn_parts = basename(full_fn).split('_') if not len(bfn_parts) == 4: _match_failure_report( full_fn, "Failure to parse: Wrong number of '_'-delimited substrings.") return # extract metadata about the file (tile, date_str, sensor, product) = bfn_parts date_pattern = data_class.Asset.Repository._datedir try: date = datetime.datetime.strptime(date_str, date_pattern).date() except Exception: verbose_out(traceback.format_exc(), 4, sys.stderr) msg = "Failure to parse date: '{}' didn't adhere to pattern '{}'." _match_failure_report(full_fn, msg.format(date_str, date_pattern)) return (product, created) = mpo.update_or_create(product=product, sensor=sensor, tile=tile, date=date, driver=driver, name=full_fn) product.save() # TODO can subtract this item from starting_keys each time and possibly save some memory and time touched_rows.add(product.pk) if created: counts['add'] += 1 verbose_out("Product added to database: " + full_fn, 5) else: counts['update'] += 1 verbose_out("Product found in database: " + full_fn, 5)
def fetch(cls, asset, tile, date): # The nassgeodata site is known to have an invalid certificate. # We don't want to clutter up the output with SSL warnings. if asset == _cdlmkii: verbose_out("Fetching not supported for cdlmkii", 2) return [] verbose_out("Fetching tile for {} on {}".format(tile, date.year), 2) query_rv = cls.query_service(asset, tile, date) if query_rv is None: verbose_out("No CDL data for {} on {}".format(tile, date.year), 2) return [] file_response = requests.get( query_rv['url'], verify=False, stream=True) with utils.make_temp_dir( prefix='fetch', dir=cls.Repository.path('stage')) as tmp_dir: fname = "{}_{}_cdl_cdl.tif".format(tile, date.year) tmp_fname = tmp_dir + '/' + fname with open(tmp_fname, 'w') as asset: asset.write(file_response.content) imgout = GeoImage(tmp_fname, True) imgout.SetNoData(0) imgout.SetMeta('GIPS_Version', gips.__version__) imgout = None shutil.copy(tmp_fname, cls.Repository.path('stage'))
def fetch(cls, asset, tile, date): """Standard Asset.fetch implementation for downloading assets.""" asset_info = cls.query_service(asset, tile, date) if asset_info is None: return [] basename, url = asset_info['basename'], asset_info['url'] basename = asset_info['basename'] url = asset_info['url'] outpath = os.path.join(cls.Repository.path('stage'), basename) with utils.error_handler("Error fetching {} from {}".format( basename, url), continuable=True): # obtain the data response = cls.Repository.managed_request(url) if response is None: return [] stage_dir = cls.Repository.path('stage') with utils.make_temp_dir(prefix='fetch', dir=stage_dir) as tmp_dn: tmp_outpath = os.path.join(tmp_dn, basename) with open(tmp_outpath, 'w') as fd: fd.write(response.read()) # verify that it is a netcdf file try: ncroot = Dataset(tmp_outpath) os.rename(tmp_outpath, outpath) except Exception as e: text = '' if e.message.endswith('Unknown file format'): token = 'Authorize NASA GESDISC DATA ARCHIVE' html = open(tmp_outpath, 'r').read(100000) if token in html: text = ('\n\nYou need to {t} \nfor your NASA ' 'EarthData login.\n').format(t=token) raise Exception(e.message + text) utils.verbose_out('Retrieved ' + basename, 2) return [outpath]
def fetch(cls, asset, tile, date): utils.verbose_out('%s: fetch tile %s for %s' % (asset, tile, date), 3) qs_rv = cls.query_service(asset, tile, date) if qs_rv is None: return [] asset_fn = qs_rv['basename'] with utils.error_handler("Error downloading from " + cls._assets[asset]['host'], continuable=True): ftp = cls.ftp_connect( asset, date) # starts chdir'd to the right directory stage_dir_fp = cls.Repository.path('stage') stage_fp = os.path.join(stage_dir_fp, asset_fn) with utils.make_temp_dir(prefix='fetchtmp', dir=stage_dir_fp) as td_name: temp_fp = os.path.join(td_name, asset_fn) utils.verbose_out("Downloading " + asset_fn, 2) with open(temp_fp, "wb") as temp_fo: ftp.retrbinary('RETR ' + asset_fn, temp_fo.write) ftp.quit() os.rename(temp_fp, stage_fp) return [stage_fp] return []
def process_indices(self, a_obj, indices): """Make the given indices from the given asset. indices' values are the keys, split by hyphen, eg {ndvi-toa': ['ndvi', 'toa']} (technically it's useless here, as no argumentated products are supported; doing it for consistency). """ # TODO another place that can be DRY'd out; several drivers process similarly verbose_out('Starting processing on {} for {}'.format( a_obj.atd_str(), indices.keys()), 4) gippy_input = {} # gippy wants p-types mapped to output filenames tempfps_to_ptypes = {} # AddFile needs a map of p-types to filenames for prod_type, pt_split in indices.items(): temp_fp = self.temp_product_filename(a_obj.sensor, prod_type) gippy_input[pt_split[0]] = temp_fp tempfps_to_ptypes[temp_fp] = prod_type prodout = gippy.algorithms.Indices( a_obj.load_image(), gippy_input, self.prep_meta(a_obj.filename)) for temp_fp in prodout.values(): archived_fp = self.archive_temp_path(temp_fp) self.AddFile(a_obj.sensor, tempfps_to_ptypes[temp_fp], archived_fp)
def _calculate_estimate(filename): val, var = cls._read_point(filename, roi, nodata) aod = numpy.nan norm = numpy.nan # Negative values don't make sense if val < 0: val = 0 if var == 0: # There is only one observation, so make up # the variance. if val == 0: var = 0.15 else: var = val / 2 if not numpy.isnan(val) and not numpy.isnan(var): aod = val / var norm = 1.0 / var utils.verbose_out('AOD: LTA-Daily = %s, %s' % (val, var), 3) return aod, norm