def __open_targz(self, fn):
assert fn.endswith('.tar.gz')
# Open tarfile
tar = tarfile.open(fn)
product_id = None
for member in tar.getnames():
if member.lower().endswith('.tif'):
product_id = '_'.join(_split(member)[-1].split('_')[:7])
break
d = {}
for member in tar.getnames():
key = _split(member)[-1].replace('%s_' % product_id, '')\
.replace('.tif', '').replace('.TIF', '')
if member.lower().endswith('.xml'):
d['.xml'] = tar.extractfile(member).read()
if member.lower().endswith('.txt') and 'MTL' in member:
d['.mtl'] = tar.extractfile(member).read()
elif member.lower().endswith('.tif'):
d[key] = member
self.tar = tar
self.product_id = product_id
self.fn = fn
self._d = d
def anu_wepp_management_mod(runs_dir):
plant_loops = set()
man_fns = glob(_join(runs_dir, '*.man'))
backup_dir = _join(runs_dir, 'original_mans')
if not _exists(backup_dir):
os.mkdir(backup_dir)
for man_fn in man_fns:
_fn = _split(man_fn)[-1]
if 'pw0' in _fn:
continue
_man_fn = _join(backup_dir, _fn)
shutil.move(man_fn, _man_fn)
for man_fn in man_fns:
_fn = _split(man_fn)[-1]
if 'pw0' in _fn:
continue
man = Management(Key=None,
ManagementFile=_fn,
ManagementDir=backup_dir,
Description='-',
Color=(0, 0, 0, 0))
with open(man_fn, 'w') as fp:
fp.write(str(man))
def __open_dir(self, fn):
fns = glob(_join(fn, '*.xml'))
assert len(fns) == 1, fns
product_id = _split(fns[0])[-1][:-4].replace('_MTL', '')
if product_id.endswith('_'):
product_id = product_id[:-1]
d = {}
for fn in glob(_join(fn, '*')):
key = _split(fn)[-1].replace('%s_' % product_id, '') \
.lower() \
.replace('.tif', '') \
.replace('qa_pixel', 'pixel_qa') \
.replace('sr_b1', 'sr_band1') \
.replace('sr_b2', 'sr_band2') \
.replace('sr_b3', 'sr_band3') \
.replace('sr_b4', 'sr_band4') \
.replace('sr_b5', 'sr_band5') \
.replace('sr_b6', 'sr_band6') \
.replace('sr_b7', 'sr_band7') \
if fn.lower().endswith('.xml'):
with open(fn, 'r') as fp:
d['.xml'] = fp.read()
elif fn.lower().endswith('.tif'):
d[key] = rasterio.open(fn)
self.product_id = product_id
self.fn = fn
self._d = d
def string_to_filepath_parts(path):
"""Break the passed path into a list of the individual parts,
e.g. /home/user/test/../something/./new.txt will return
['home', 'user', 'something', 'new.txt']
Args:
path (str): Path to split
Returns:
list: List of sections of path
"""
from os.path import split as _split
path = string_to_filepath(path)
dirs = []
(root, part) = _split(path)
dirs.append(part)
while len(root) > 0:
(root, part) = _split(root)
if len(part) > 0:
dirs.insert(0, part)
else:
break
return dirs
def is_processed(fn):
global out_dir
_fn = _split(fn)[-1].split('-')[0]
# res = glob(_join(out_dir, '{}_*_{}_{}_*_{}_{}'
# .format(_fn[:4], _fn[4:10], _fn[10:18], _fn[18:20], _fn[20:22])))
res = glob(_join(out_dir, fn.replace('.tar.gz', '') + '*.csv'))
return len(res) > 0 or _exists(
_join(out_dir, '.{}'.format(_split(fn.replace('.tar.gz', ''))[-1])))
def process_scene(scn_fn, verbose=True):
global models, out_dir, sf, bbox, sf_feature_properties_key, sf_feature_properties_delimiter
assert '.tar.gz' in scn_fn
if verbose:
print(scn_fn, out_dir)
print('extracting...')
scn_path = scn_fn.replace('.tar.gz', '')
if _exists(SCRATCH):
scn_path = _join(SCRATCH, _split(scn_path)[-1])
extract(scn_fn, scn_path)
# Load and crop LandSat Scene
print('load')
_ls = LandSatScene(scn_path)
try:
print('clip')
ls = _ls.clip(bbox, out_dir)
except:
ls = None
_ls = None
shutil.rmtree(scn_path)
Path(_join(out_dir, '.{}'.format(_split(scn_path)[-1]))).touch()
raise
_ls.dump_rgb(_join(ls.basedir, 'rgb.tif'), gamma=1.5)
print('ls.basedir', ls.basedir)
# Build biomass model
bio_model = BiomassModel(ls, models)
# Export grids
print('exporting grids')
bio_model.export_grids(biomass_dir=_join(ls.basedir, 'biomass'))
# Analyze pastures
print('analyzing pastures')
res = bio_model.analyze_pastures(sf, sf_feature_properties_key, sf_feature_properties_delimiter)
# get a summary dictionary of the landsat scene
print('compiling summary')
ls_summary = ls.summary_dict()
print('reprojecting scene')
scn_dir = _join(out_dir, _ls.product_id)
reproject_scene(scn_dir)
# ls = None
# _ls = None
# shutil.rmtree(scn_path)
return dict(res=res, ls_summary=ls_summary)
def _run_wepp_hillslopes(self):
# TODO: Test this function
global USE_MULTIPROCESSING
runs_dir = self.runs_dir
hillslope_runs = glob(_join(runs_dir, 'p*.run'))
hillslope_runs = [run for run in hillslope_runs if 'pw' not in run]
if USE_MULTIPROCESSING:
pool = ThreadPoolExecutor(NCPU)
futures = []
for hillslope_run in hillslope_runs:
run_fn = _split(hillslope_run)[-1]
wepp_id = run_fn.replace('p', '').replace('.run', '')
assert isint(wepp_id), wepp_id
if USE_MULTIPROCESSING:
futures.append(pool.submit(lambda p: run_hillslope(*p), (int(wepp_id), runs_dir)))
futures[-1].add_done_callback(oncomplete)
else:
status, _id, elapsed_time = run_hillslope(int(wepp_id), runs_dir)
assert status
print(' {} completed run in {}s\n'.format(_id, elapsed_time))
if USE_MULTIPROCESSING:
wait(futures, return_when=FIRST_EXCEPTION)
def __init__(self):
# { attr, raster_file_path}
catalog = glob(_join(_asris_grid_raster_dir, '*'))
catalog = [path for path in catalog if os.path.isdir(path)]
catalog = {_split(path)[-1]: path for path in catalog}
self.catalog = catalog
# { attr, raster_attribute table}
rats = {}
for var, path in catalog.items():
fn = _join(path + '.json')
if not _exists(fn):
continue
with open(fn) as fp:
info = json.load(fp)
if 'rat' not in info:
continue
rows = info['rat']['row']
d = {}
for row in rows:
row = row['f']
d[row[0]] = row[-1]
rats[var] = d
self.rats = rats
def reprocess_scene(scn_fn):
p = subprocess.Popen(['python3', 'reprocess_scene.py', cfg_fn, scn_fn],
stderr=subprocess.PIPE, stdout=subprocess.PIPE)
p.wait()
scn_path = scn_fn.replace('.tar.gz', '')
if _exists(SCRATCH):
scn_path = _join(SCRATCH, _split(scn_path)[-1])
if _exists(scn_path):
shutil.rmtree(scn_path)
def process_scene(scn_fn):
p = subprocess.Popen(['python3', 'process_scene.py', cfg_fn, scn_fn],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
ret = p.communicate()
print(ret)
scn_path = scn_fn.replace('.tar.gz', '')
if _exists(SCRATCH):
scn_path = _join(SCRATCH, _split(scn_path)[-1])
if _exists(scn_path):
shutil.rmtree(scn_path)
def readpar(par):
fid = open(par)
state = par[:2]
desc = fid.readline().strip()
line1 = _p(fid.readline())
assert len(line1) == 4
latitude, longitude, years, type = line1
line2 = _p(fid.readline())
assert len(line2) == 3, str(line2)
elevation, tp5, tp6 = line2
fid.close()
return '"%s"' % state, '"%s"' % desc, '"%s"' % _split(par)[-1], latitude, longitude, years, type, elevation, tp5, tp6
def build_catalog(directory):
fns = glob(_join(directory, '*.gz'))
catalog = []
for fn in fns:
product_id = _split(fn)[-1].split('-')[0]
satellite = int(product_id[2:4])
wrs_path = int(product_id[4:7])
wrs_row = int(product_id[7:10])
_date = product_id[10:18]
year, month, day = int(_date[:4]), int(_date[4:6]), int(_date[6:])
catalog.append(tuple([satellite, wrs_path, wrs_row, year, month, day]))
return catalog
def tar_gz_is_valid(tar_fn):
scn_path = _join(SCRATCH, _split(tar_fn)[-1].replace('.tar.gz', ''))
is_valid = False
try:
tar = tarfile.open(tar_fn)
tar.extractall(path=scn_path)
tar.close()
is_valid = True
except:
pass
try:
shutil.rmtree(scn_path)
except:
pass
return is_valid
def get_plant_loop_names(runs_dir):
plant_loops = set()
man_fns = glob(_join(runs_dir, '*.man'))
for man_fn in man_fns:
_fn = _split(man_fn)[-1]
if 'pw0' in _fn:
continue
man = Management(Key=None,
ManagementFile=_fn,
ManagementDir=runs_dir,
Description='-',
Color=(0, 0, 0, 0))
plant_loops.add(man.plants[0].name)
return list(plant_loops)
def run_watershed(runs_dir, output_dir):
print('runs_dir', runs_dir)
t0 = time()
cmd = [os.path.abspath(wepp_exe)]
assert exists(_join(runs_dir, 'pw0.str'))
assert exists(_join(runs_dir, 'pw0.chn'))
assert exists(_join(runs_dir, 'pw0.imp'))
assert exists(_join(runs_dir, 'pw0.man'))
assert exists(_join(runs_dir, 'pw0.slp'))
assert exists(_join(runs_dir, 'pw0.cli'))
assert exists(_join(runs_dir, 'pw0.sol'))
assert exists(_join(runs_dir, 'pw0.run'))
_run = open(_join(runs_dir, 'pw0.run'))
_log = open(_join(runs_dir, 'pw0.err'), 'w')
p = subprocess.Popen(cmd,
stdin=_run,
stdout=_log,
stderr=_log,
cwd=runs_dir)
p.wait()
_run.close()
_log.close()
log_fn = _join(runs_dir, 'pw0.err')
for fn in glob(_join(runs_dir, '*.out')):
dst_path = _join(output_dir, _split(fn)[1])
shutil.move(fn, dst_path)
with open(_join(runs_dir, 'pw0.err')) as fp:
stdout = fp.read()
if 'WEPP COMPLETED WATERSHED SIMULATION SUCCESSFULLY' in stdout:
return True, time() - t0
raise Exception(
'Error running wepp for watershed \nSee <a href="browse/wepp/runs/pw0.err">%s</a>'
% log_fn)
def __init__(self, identifier):
_identifier = _split(identifier)
self.identifier = _identifier[-1]
path = identifier
assert _exists(path)
self.path = path
self.file = file = SD(path, SDC.READ)
_variables = {}
for short_name in file.datasets().keys():
band = file.select(short_name)
attrs = band.attributes()
if 'long_name' in attrs:
_variables[attrs['long_name']] = short_name
elif 'QA description' in attrs:
_variables['QA'] = short_name
else:
raise NotImplementedError()
self._variables = _variables
def _get_lib_path_no_extension(filepath, splitext=_splitext):
"""Internal function to find file when given basepath but
no file extension.
Scan the directory of the file and try to match the head of
the filepath to an entry in that directory.
@param filepath: a filename with extension but no basename
To make function not require xl param, just use
xl = newExcel(False, False) and close at the end.
@type filepath: str
@rtype: str
"""
base, head = _split(filepath)
for entry in _listdir(base):
filename, ext = splitext(entry)
# if not ext, then we have a dir, not filename
# test ext first to short circuit non-files
if ext and filename == head:
return '\\'.join((base, entry))
raise FileNotFoundError("File '%s' not found via no-extension search" % filepath)
def __init__(self, loc_path):
cfg_fn = glob(_join(loc_path, '*.yaml'))
assert len(cfg_fn) == 1, cfg_fn
cfg_fn = cfg_fn[0]
with open(cfg_fn) as fp:
yaml_txt = fp.read()
yaml_txt = yaml_txt.replace('{GEODATA}', GEODATA)
_d = yaml.safe_load(yaml_txt)
self.cfg_fn = cfg_fn
self._d = _d
self.rangesat_dir, self.location = _split(loc_path)
self.loc_path = loc_path
geojson = glob('{}/*.geojson'.format(loc_path))
assert len(geojson) == 1, geojson
self.geojson = geojson[0]
sf_fn = _join(loc_path, _d['sf_fn'])
sf_feature_properties_key = _d['sf_feature_properties_key']
sf_fn = os.path.abspath(sf_fn)
sf = fiona.open(sf_fn, 'r')
reverse_key = self.reverse_key
pastures = {}
for feature in sf:
properties = feature['properties']
key = properties[sf_feature_properties_key].replace(' ', '_')
pasture, ranch = key.split(self.key_delimiter)
if reverse_key:
ranch, pasture = pasture, ranch
if ranch.lower() not in [p.lower() for p in pastures]:
pastures[ranch] = set()
pastures[ranch].add(pasture)
self.pastures = pastures
def readpar(par):
fid = open(par)
desc = fid.readline().strip()
if "United States" in desc:
state = 'US'
elif "United Kingdom" in desc:
state = 'UK'
else:
state = desc.split()[0]
line1 = _p(fid.readline())
assert len(line1) == 4
latitude, longitude, years, type = line1
line2 = _p(fid.readline())
assert len(line2) == 3, str(line2)
elevation, tp5, tp6 = line2
fid.close()
return '"%s"' % state, '"%s"' % desc.replace('"', ''), '"%s"' % _split(
par)[-1], latitude, longitude, years, type, elevation, tp5, tp6
def __init__(self,
state,
desc,
par,
latitude,
longitude,
years,
_type,
elevation,
tp5,
tp6,
_distance=None):
par0, par1 = _split(par)
self.state = state
self.par = par1
self.latitude = latitude
self.longitude = longitude
self.years = years
self.type = _type
self.elevation = elevation
self.tp5 = tp5
self.tp6 = tp6
self.distance = None
self.lat_distance = None
self.rank = None
self.id = par.replace('.par', '')
#assert len(self.id) == 6
self.desc = desc.split(str(self.id))[0].strip()
if par0 == '':
self.parpath = _join(_stations_dir, par)
else:
self.parpath = par
assert _exists(self.parpath)
def dump(self, data, dst_fn, nodata=-9999, dtype=rasterio.float32):
"""
utility method to export arrays
with the same projection and size as the
landsat scenes as geotiffs
"""
assert _exists(_split(dst_fn)[0])
if isinstance(data, np.ma.core.MaskedArray):
data.fill_value = nodata
_data = data.filled()
else:
_data = data
with rasterio.Env():
profile = self._d['pixel_qa'].profile
profile.update(dtype=rasterio.float32,
count=1,
nodata=nodata,
compress='lzw')
with rasterio.open(dst_fn, 'w', **profile) as dst:
dst.write(_data.astype(rasterio.float32), 1)
climate = Climate.getInstance(wd)
if build_climates:
log_print('building climate')
if cli_mode == 'observed':
log_print('building observed')
if 'linveh' in wd:
climate.climate_mode = ClimateMode.ObservedDb
climate.climate_spatialmode = ClimateSpatialMode.Multiple
climate.input_years = 21
climate.lock()
lng, lat = watershed.centroid
cli_path = lvdm.closest_cli(lng, lat)
_dir, cli_fn = _split(cli_path)
shutil.copyfile(cli_path, _join(climate.cli_dir, cli_fn))
climate.cli_fn = cli_fn
par_path = lvdm.par_path
_dir, par_fn = _split(par_path)
shutil.copyfile(par_path, _join(climate.cli_dir, par_fn))
climate.par_fn = par_fn
sub_par_fns = {}
sub_cli_fns = {}
for topaz_id, ss in watershed._subs_summary.items():
log_print(topaz_id)
lng, lat = ss.centroid.lnglat
cli_path = lvdm.closest_cli(lng, lat)
def run_observed(self, prn_fn, cli_fn='wepp.cli', verbose=False):
if verbose:
print("running observed")
if self.cliver not in ['5.2', '5.3', '5.3.2']:
raise NotImplementedError('Cligen version must be greater than 5')
if self.cliver == '5.2':
if IS_WINDOWS:
raise NotImplementedError(
'Cligen52.exe is not available on Windows')
else:
cligen_bin = _join(_bin_dir, 'cligen52')
elif self.cliver == '5.3':
if IS_WINDOWS:
cligen_bin = _join(_bin_dir, 'cligen53.exe')
else:
cligen_bin = _join(_bin_dir, 'cligen53')
elif self.cliver == '5.3.2':
if IS_WINDOWS:
cligen_bin = _join(_bin_dir, 'cligen532.exe')
else:
cligen_bin = _join(_bin_dir, 'cligen532')
assert _exists(cligen_bin)
assert cli_fn.endswith('.cli')
station_meta = self.station
# no prism adjustment is specified
# just copy the par into the working directory
par_fn = _join(self.wd, station_meta.par)
if not _exists(par_fn):
shutil.copyfile(station_meta.parpath, par_fn)
assert _exists(par_fn)
_, par = os.path.split(par_fn)
# change to working directory
cli_dir = self.wd
# delete cli file if it exists
if _exists(_join(cli_dir, cli_fn)):
os.remove(_join(cli_dir, cli_fn))
cmd = [
cligen_bin,
"-i%s" % par,
"-O%s" % prn_fn,
"-o%s" % cli_fn, "-t6", "-I2"
]
if verbose:
print(cmd)
# run cligen
_log = open(
_join(cli_dir, "cligen_{}.log".format(_split(cli_fn)[-1][:-4])),
"w")
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=_log,
stderr=_log,
cwd=cli_dir)
p.wait()
_log.close()
assert _exists(_join(cli_dir, cli_fn))
def xlBook2(filepath=None, new=False, visible=True, search=False, xl=None):
"""Get win32com workbook object from filepath.
If workbook is open, get active object.
If workbook is not open, create a new instance of
xl and open the workbook in that instance.
If filename is not found, see if user specified
default filename error behavior as returning new
workbook. If not, raise error. If so, return new workbook
Warning: returns error in some circumstances if dialogs or
certain areas like formula bar in the desired Excel instance
have focus.
@param filepath: valid filepath
@type filepath: str | None
@param visible: xl instance visible to user?
turn off to do heavy processing before showing
@type visible: bool
@param new: open in a new window
@type new: bool
@return: the newly opened xl workbook instance
@rtype: (typehint.ExcelApplication, typehint.ExcelWorkbook)
Update 1/15/2014- Lots of refactoring to make it really clean and such.
Or so I tried.
Update 1/29/2014- this function is now converted to abstract internal function.
Interfaced moved to new function with same name.
This function still contains logic.
Update 1/31/2014- renamed function xlBook2, now public.
"""
if xl is None:
xl = Excel(new, visible)
if not filepath:
wb = __ensure_wb(xl)
return xl, wb
_base, name = _split(filepath)
no_ext_name, ext = _splitext(name)
# First try to see if passed name of open workbook
# xl can be a pain, so try with and without ext.
wbs = xl.Workbooks
possible_names = (
filepath.lstrip("\\/"),
no_ext_name,
name
)
if wbs.Count:
for fname in possible_names:
try:
wb = wbs(fname)
except:
continue
else:
v_print("\'%s\' found, returning existing workbook." % filepath)
wb.Activate()
return xl, wb
# Workbook wasn't open, get filepath and open it.
# This may take a *long* time.
try:
if search:
v_print("Searching for file...")
filepath = getFullFilename(filepath, hint=xl.DefaultFilePath)
except FileNotFoundError as e:
# cleanup if filepath wasn't found.
if new:
xl.Quit()
else:
xl.Visible = True
raise xlLibError("Couldn't find path '%s', check that it is correct." % filepath) from e
try:
wb = wbs.Open(filepath, Notify=False)
except:
if new:
xl.Quit()
else:
xl.Visible = True
else:
v_print("Filename \'%s\' found.\nReturning newly opened workbook." % filepath)
wb.Activate()
return xl, wb
raise xlLibError("Unknown error occurred. \nCheck filename: %s "
"If the target file is open, ensure\nno dialogs are open." % filepath)
models.append(ModelPars(_m['name'], _satellite_pars))
# open shape file and determine the bounds
sf_fn = _d['sf_fn']
sf_fn = os.path.abspath(sf_fn)
sf = fiona.open(sf_fn, 'r')
bbox = get_sf_wgs_bounds(sf_fn)
landsat_scene_directory = _d['landsat_scene_directory']
wrs_blacklist = _d.get('wrs_blacklist', None)
sf_feature_properties_key = _d.get('sf_feature_properties_key', 'key')
sf_feature_properties_delimiter = _d.get('sf_feature_properties_delimiter', '+')
out_dir = _d['out_dir']
scene_fn = sys.argv[-1]
scn_bounds = get_gz_scene_bounds(scene_fn)
if not bounds_contain(bbox, scn_bounds):
print('bounds do not intersect', bbox, scn_bounds)
Path(_join(out_dir, '.{}'.format(_split(scene_fn.replace('.tar.gz', ''))[-1]))).touch()
sys.exit()
res = process_scene(scene_fn)
prefix = os.path.basename(os.path.normpath(scene_fn)).replace('.tar.gz', '')
dump_pasture_stats([res], _join(out_dir, '%s_pasture_stats.csv' % prefix))
def getFullFilename(path, hint=None):
""" Function to get full library path. Figure out
what's in the path iteratively, based on 3 common scenarios.
@param path: a filepath or filename
@type path: str
@param hint: the first directory tree in which to search for the file
@type hint: str
@return: full library path to existing file.
@rtype: str
Try to find the path by checking for three common cases:
1. filename with extension
- base
- no base
2. filename with only base
2.1 partially qualified directory name
2.2 fully qualified directory name
3. neither one
Build list of folders to search by calling first helper function.
Update 1/16/2014- xl nonsense gone
I moved the algorithm for executing the search to an inlined dispatch function
that receives all the relative args, for the sake of making this function
cleaner, but I'm not sure if that level of indirection just makes
everything even worse. Having it defined within this function allows
it to access path, etc variables without having to explicitly call them.
In all, there is much less text in the areas in which the dispatch is called.
"""
path = path.replace('/', '\\') # Normalize sep type
# Was path already good?
if _exists(path):
return path
# Begin process of finding file
search_dirs = _lib_path_search_dir_list_builder(hint)
base, name = _split(path)
ext = _splitext(name)[1]
# Most likely- given extension.
# no need to check for case of fully qualified basename.
# an existing file with a fully qualified base name and extension would
# be caught by earlier _exists()
if ext:
if base:
v_print('\nPartially qualified filename \'', path, "\' given, searching for file...")
return _get_lib_path_parital_qualname(name, base, search_dirs)
# else
v_print("\nNo directory given for \'", path, "\', scanning for file...")
return _get_lib_path_no_basename(path, search_dirs)
# Next, given filename with base, but no extension
elif base:
drive, _tail = _splitdrive(base)
# fully qualified base, just check the dir for matching name
if drive:
v_print("\nNo file extension given for \'", path, "\', scanning for file...")
return _get_lib_path_no_extension(path)
# partially qualified base, search dirs. I don't think this works well (at all).
# I don't think I managed to get a working unittest for it.
else:
v_print("\nAttempting to find partially qualified name \'", path, "\' ...")
return _get_lib_path_parital_qualname(name, base, search_dirs)
# Finally, user gave no context- no base or extension.
# Try really hard to find it anyway.
else:
v_print("\nNo context given for filename, scanning for file.\nIf you give a full filepath, you wouldn't \nhave to wait for the long search.")
return _get_lib_path_no_ctxt(path, search_dirs)
# noinspection PyUnreachableCode
raise SystemExit("Unreachable code reached: fix module olutils")
try:
ash_out = ash_post.ash_out
except:
ash_out = None
raise
name = ron.name
# _wd = _split(wd)[-1].split('_')
# if _wd[3][0] in 'BWG' or _wd[3].startswith('Me'):
# indx = 4
# else:
# indx = 3
scenario = 'sbs' # '_'.join(_wd[indx:])
watershed = _split(wd)[-1] # '_'.join(_wd[:indx])
run_descriptors = [('ProjectName', name), ('Scenario', scenario), ('Watershed', watershed)]
loss = Wepp.getInstance(wd).report_loss()
hill_rpt = HillSummary(loss, subs_summary=subcatchments_summary, ash_out=ash_out)
chn_rpt = ChannelSummary(loss, chns_summary=channels_summary)
out_rpt = OutletSummary(loss)
sed_del = SedimentDelivery(wd)
hill_rpt.write(fp_hill, write_header=write_header, run_descriptors=run_descriptors)
chn_rpt.write(fp_chn, write_header=write_header, run_descriptors=run_descriptors)
out_rpt.write(fp_out, write_header=write_header, run_descriptors=run_descriptors)
sed_del.write(fp_sd, write_header=write_header, run_descriptors=run_descriptors)
print(wd)
write_header = i == 0
ron = Ron.getInstance(wd)
subcatchments_summary = {
_d['meta']['topaz_id']: _d
for _d in ron.subs_summary()
}
channels_summary = {
_d['meta']['topaz_id']: _d
for _d in ron.chns_summary()
}
name = ron.name
_wd = _split(wd)[-1]
for scn in scenarios:
lt_date, scn_desc = scn.split('*')
_wd = _wd.replace(lt_date, '')
if scn_desc in _wd:
scenario = scn_desc
_wd = _wd.replace(scn_desc, '')
watershed = _wd[1:-1]
run_descriptors = [('ProjectName', name), ('Scenario', scenario),
('Watershed', watershed)]
loss = Wepp.getInstance(wd).report_loss()
hill_rpt = HillSummary(loss,
class_fractions=True,
import json
from glob import glob
from os.path import join as _join
from os.path import split as _split
import os
from subprocess import Popen, check_output
_asris_grid_raster_dir = '/geodata/au/asris/'
catalog = glob(_join(_asris_grid_raster_dir, '*'))
catalog = [path for path in catalog if os.path.isdir(path)]
catalog = {_split(path)[-1]: path for path in catalog}
for ds, path in catalog.items():
print(ds)
js = check_output('gdalinfo -json ' + _join(path, ds), shell=True)
info = json.loads(js.decode())
with open(_join(_asris_grid_raster_dir, ds + '.json'), 'w') as fp:
json.dump(info, fp, indent=4, sort_keys=True, allow_nan=False)
def export_band(self,
band,
as_float=True,
compress=True,
out_dir=None,
force_utm_zone=None):
if out_dir is None:
out_dir = _split(self.path)[0]
if as_float:
_data = getattr(self, band)
dtype = np.float32
else:
add_offset = self._get_band_add_offset(band)
scale_factor = self._get_band_scale_factor(band)
dtype = self._get_band_dtype(band)
_data = getattr(self, band)
_data = np.ma.array((_data / scale_factor) - add_offset,
dtype=dtype)
fill_value = self._get_band_fill_value(band)
gdal_type = {
np.float32: gdal.GDT_Float32,
np.float64: gdal.GDT_Float64,
np.int16: gdal.GDT_Int16,
np.uint8: gdal.GDT_Byte
}[dtype]
driver = gdal.GetDriverByName('GTiff')
fname = tmp_fname = '{}-{}.tif'.format(self.identifier[:-4], band)
fname = _join(out_dir, fname)
if compress:
tmp_fname = fname[:-4] + '.tmp.tif'
tmp_fname = _join(out_dir, tmp_fname)
if _exists(tmp_fname):
os.remove(tmp_fname)
ds = driver.Create(tmp_fname, self.nrows, self.ncols, 1, gdal_type)
ds.SetGeoTransform(self.transform)
srs = osr.SpatialReference()
srs.SetUTM(self.utm_zone, (0, 1)[self.is_north])
srs.SetWellKnownGeogCS(self.geog_cs)
ds.SetProjection(srs.ExportToWkt())
_band = ds.GetRasterBand(1)
_band.WriteArray(_data)
if fill_value is not None:
_band.SetNoDataValue(fill_value)
ds = None
if force_utm_zone is not None:
if int(force_utm_zone) == int(self.utm_zone):
tmp2_fname = fname[:-4] + '.tmp2.tif'
utm_proj4 = "+proj=utm +zone={zone} +{hemisphere} +datum=WGS84 +ellps=WGS84" \
.format(zone=force_utm_zone, hemisphere=('south', 'north')[self.is_north])
cmd = [
'gdal_translate', '-t_srs', '"{}"'.format(utm_proj4),
tmp_fname, tmp2_fname
]
_log = open(tmp2_fname + '.err', 'w')
p = Popen(cmd, stdout=_log, stderr=_log)
p.wait()
_log.close()
if _exists(tmp2_fname):
os.remove(tmp2_fname + '.err')
os.remove(tmp_fname)
tmp_fname = tmp2_fname
if compress:
cmd = [
'gdal_translate', '-co', 'compress=DEFLATE', '-co', 'zlevel=9',
tmp_fname, fname
]
_log = open(fname + '.err', 'w')
p = Popen(cmd, stdout=_log, stderr=_log)
p.wait()
_log.close()
if _exists(fname):
os.remove(fname + '.err')
os.remove(tmp_fname)
return fname
def is_processed(fn):
global out_dir
_fn = _split(fn)[-1].split('-')[0]
res = glob(_join(out_dir, '{}_*_{}_{}_*_{}_{}'
.format(_fn[:4], _fn[4:10], _fn[10:18], _fn[18:20], _fn[20:22])))
return len(res) > 0
def merge_and_crop(self,
others,
bands,
bbox,
as_float=False,
out_dir=None,
verbose=True):
ul_x, ul_y, lr_x, lr_y = bbox
assert ul_x < lr_x
assert ul_y < lr_y
# determine UTM coordinate system of top left corner
ul_e, ul_n, utm_number, utm_letter = utm.from_latlon(latitude=ul_y,
longitude=ul_x)
# bottom right
lr_e, lr_n, _, _ = utm.from_latlon(latitude=ul_y,
longitude=ul_x,
force_zone_number=utm_number,
force_zone_letter=utm_letter)
utm_proj4 = "+proj=utm +zone={zone} +{hemisphere} +datum=WGS84 +ellps=WGS84" \
.format(zone=utm_number, hemisphere=('south', 'north')[ul_y > 0])
if out_dir is None:
out_dir = _split(self.path)[0]
acquisition_date = self.acquisition_date
sat = self.sat
proj4 = self.proj4
for other in others:
assert acquisition_date == other.acquisition_date, (
acquisition_date, other.acquisition_date)
assert sat == other.sat, (sat, other.sat)
for band in bands:
srcs = []
srcs.append(
self.export_band(band,
as_float=as_float,
out_dir=out_dir,
proj4=proj4))
for other in others:
srcs.append(
other.export_band(band,
as_float=as_float,
out_dir=out_dir,
proj4=proj4))
vrt_fn = self.identifier.split('.')
vrt_fn[2] = 'XXXXXX'
vrt_fn[-1] = 'vrt'
vrt_fn.insert(-1, '_{}'.format(band))
vrt_fn = '.'.join(vrt_fn)
vrt_fn = _join(out_dir, vrt_fn)
fname = vrt_fn[:-4] + '.tif'
cmd = ['gdalbuildvrt', vrt_fn] + srcs
_log = open(vrt_fn + '.err', 'w')
p = Popen(cmd, stdout=_log, stderr=_log)
p.wait()
_log.close()
if _exists(vrt_fn):
os.remove(vrt_fn + '.err')
cmd = [
'gdal_translate', '-co', 'compress=DEFLATE', '-co', 'zlevel=9',
vrt_fn, fname
]
_log = open(vrt_fn + '.err', 'w')
p = Popen(cmd, stdout=_log, stderr=_log)
p.wait()
_log.close()
if _exists(fname):
os.remove(fname + '.err')
for src in srcs:
os.remove(src)
os.remove(vrt_fn)
hillslope_runs = glob(_join(runs_dir, 'p*.run'))
hillslope_runs = [run for run in hillslope_runs if 'pw' not in run]
print('cleaning output dir')
if exists(output_dir):
shutil.rmtree(output_dir)
os.mkdir(output_dir)
if USE_MULTIPROCESSING:
pool = ThreadPoolExecutor(NCPU)
futures = []
for hillslope_run in hillslope_runs:
run_fn = _split(hillslope_run)[-1]
wepp_id = run_fn.replace('p', '').replace('.run', '')
assert isint(wepp_id), wepp_id
if USE_MULTIPROCESSING:
futures.append(
pool.submit(lambda p: run_hillslope(*p),
(int(wepp_id), runs_dir)))
futures[-1].add_done_callback(oncomplete)
else:
status, _id, elapsed_time = run_hillslope(int(wepp_id), runs_dir)
assert status
print(' {} completed run in {}s\n'.format(_id, elapsed_time))
if USE_MULTIPROCESSING:
wait(futures, return_when=FIRST_EXCEPTION)