def handle(self, shapefile, code, **options):
watersheds = GisIO(filename=shapefile)
huc8s = get_huc8(code)
for id, watershed in list(watersheds.items()):
if watershed.huc_code not in huc8s:
continue
basin = Basin.objects.find(watershed.huc_code)
ident = basin.primary_identifier
ident.name = watershed.huc_name
ident.save()
def load_data(basin,
elem,
syear=1950,
eyear=curyear,
inactive=False,
years=30):
syear = int(syear)
eyear = int(eyear)
inactive = bool(inactive)
years = int(years)
if not basin.isdigit() or len(basin) != 8:
from climata.huc8 import get_huc8
basin = get_huc8(basin)
sites = StationMetaIO(
basin=basin,
start_date='%s-01-01' % syear,
end_date='%s-12-31' % eyear,
parameter=elem,
meta=ALL_META_FIELDS,
)
include_sites = []
seen_auths = set()
for site in sites:
start, end = site.valid_daterange[elem]
exclude = False
if end.year < eyear - years and not inactive:
sys.stderr.write("Site not active: %s (last %s)" %
(site.name, end.date()))
sys.stderr.write("\n")
exclude = True
elif end.year - start.year + 1 < years:
sys.stderr.write("Not enough data: %s (%s years)" %
(site.name, end.year - start.year))
sys.stderr.write("\n")
exclude = True
if exclude:
continue
include_sites.append(site)
for auth in site.sids.keys():
seen_auths.add(auth)
# Sort sites by longitude
include_sites = sorted(include_sites, key=lambda s: s.longitude)
seen_auths = sorted(seen_auths)
def get_val(site, field):
if hasattr(site, field):
return getattr(site, field)
else:
return site.sids.get(field, "")
def header(field):
vals = [get_val(site, field) for site in include_sites]
sys.stdout.write(",".join([field + ":"] + list(map(str, vals))))
sys.stdout.write("\n")
header("name")
for auth in seen_auths:
header(auth)
header("latitude")
header("longitude")
for year in range(syear, eyear + 1):
load_year_data(basin, elem, year, include_sites)
def load_sites(*basin_ids):
"""
Load metadata for all sites in given basin codes.
"""
# Resolve basin ids to HUC8s if needed
basins = []
for basin in basin_ids:
if basin.isdigit() and len(basin) == 8:
basins.append(basin)
else:
from climata.huc8 import get_huc8
basins.extend(get_huc8(basin))
# Load sites with data since 1900
sites = StationMetaIO(
basin=basins,
parameter=list(elems.keys()),
start_date='1900-01-01',
end_date=date.today(),
meta=ALL_META_FIELDS,
)
# Load all sites (to get sites without data)
seen_sites = [site.uid for site in sites]
nodata_sites = [
site for site in StationMetaIO(basin=basins)
if site.uid not in seen_sites
]
# Determine the following from the site lists:
seen_auths = set() # Which authority codes are actually used by any site
seen_elems = set() # Which elems actually have data in any site
ranges = {} # The overall period of record for each site
for site in sites:
for auth in site.sids.keys():
seen_auths.add(auth)
start, end = None, None
for elem in site.valid_daterange:
s, e = site.valid_daterange[elem]
seen_elems.add(elem)
if s is None or e is None:
continue
if start is None or s < start:
start = s
if end is None or e > end:
end = e
ranges[site.uid] = [start, end]
# Check for authority codes that might not be in sites with data
for site in nodata_sites:
for auth in site.sids.keys():
seen_auths.add(auth)
# Print CSV headers (FIXME: use CsvFileIO for this?)
seen_auths = sorted(seen_auths)
seen_elems = sorted(seen_elems)
print(",".join(
['ACIS uid', 'name']
+ seen_auths
+ ['latitude', 'longitude', 'start', 'end', 'years']
+ [elems[elem]['desc'] for elem in seen_elems]
))
# Print sites with data
for site in sites:
# Determine if elems are available for entire period or shorter range
start, end = ranges[site.uid]
if start and end:
years = end.year - start.year + 1
elem_ranges = []
for elem in seen_elems:
estart, eend = site.valid_daterange[elem]
if estart is None:
erange = ""
elif estart == start and eend == end:
erange = "period"
else:
erange = "%s to %s" % (estart.date(), eend.date())
elem_ranges.append(erange)
# Output CSV row
print(",".join(map(
str,
[site.uid, site.name]
+ [site.sids.get(auth, "") for auth in seen_auths]
+ [site.latitude, site.longitude]
+ [start.date(), end.date(), years]
+ elem_ranges
)))
# Print CSV rows for sites without data
for site in nodata_sites:
print(",".join(map(
str,
[site.uid, site.name]
+ [site.sids.get(auth, "") for auth in seen_auths]
+ [site.latitude, site.longitude]
+ ["NO DATA"]
)))
def load_sites(*basin_ids):
"""
Load metadata for all sites in given basin codes.
"""
# Resolve basin ids to HUC8s if needed
basins = []
for basin in basin_ids:
if basin.isdigit() and len(basin) == 8:
basins.append(basin)
else:
from climata.huc8 import get_huc8
basins.extend(get_huc8(basin))
# Load sites with data since 1900
sites = StationMetaIO(
basin=basins,
parameter=list(elems.keys()),
start_date='1900-01-01',
end_date=date.today(),
meta=ALL_META_FIELDS,
)
# Load all sites (to get sites without data)
seen_sites = [site.uid for site in sites]
nodata_sites = [
site for site in StationMetaIO(basin=basins)
if site.uid not in seen_sites
]
# Determine the following from the site lists:
seen_auths = set() # Which authority codes are actually used by any site
seen_elems = set() # Which elems actually have data in any site
ranges = {} # The overall period of record for each site
for site in sites:
for auth in site.sids.keys():
seen_auths.add(auth)
start, end = None, None
for elem in site.valid_daterange:
s, e = site.valid_daterange[elem]
seen_elems.add(elem)
if s is None or e is None:
continue
if start is None or s < start:
start = s
if end is None or e > end:
end = e
ranges[site.uid] = [start, end]
# Check for authority codes that might not be in sites with data
for site in nodata_sites:
for auth in site.sids.keys():
seen_auths.add(auth)
# Print CSV headers (FIXME: use CsvFileIO for this?)
seen_auths = sorted(seen_auths)
seen_elems = sorted(seen_elems)
print(",".join(['ACIS uid', 'name'] + seen_auths +
['latitude', 'longitude', 'start', 'end', 'years'] +
[elems[elem]['desc'] for elem in seen_elems]))
# Print sites with data
for site in sites:
# Determine if elems are available for entire period or shorter range
start, end = ranges[site.uid]
if start and end:
years = end.year - start.year + 1
elem_ranges = []
for elem in seen_elems:
estart, eend = site.valid_daterange[elem]
if estart is None:
erange = ""
elif estart == start and eend == end:
erange = "period"
else:
erange = "%s to %s" % (estart.date(), eend.date())
elem_ranges.append(erange)
# Output CSV row
print(",".join(
map(str, [site.uid, site.name] +
[site.sids.get(auth, "")
for auth in seen_auths] + [site.latitude, site.longitude] +
[start.date(), end.date(), years] + elem_ranges)))
# Print CSV rows for sites without data
for site in nodata_sites:
print(",".join(
map(str, [site.uid, site.name] +
[site.sids.get(auth, "") for auth in seen_auths] +
[site.latitude, site.longitude] + ["NO DATA"])))
def load_data(basin, elem, syear=1950, eyear=curyear,
inactive=False, years=30):
syear = int(syear)
eyear = int(eyear)
inactive = bool(inactive)
years = int(years)
if not basin.isdigit() or len(basin) != 8:
from climata.huc8 import get_huc8
basin = get_huc8(basin)
sites = StationMetaIO(
basin=basin,
start_date='%s-01-01' % syear,
end_date='%s-12-31' % eyear,
parameter=elem,
meta=ALL_META_FIELDS,
)
include_sites = []
seen_auths = set()
for site in sites:
start, end = site.valid_daterange[elem]
exclude = False
if end.year < eyear - years and not inactive:
sys.stderr.write("Site not active: %s (last %s)" % (
site.name, end.date()
))
sys.stderr.write("\n")
exclude = True
elif end.year - start.year + 1 < years:
sys.stderr.write("Not enough data: %s (%s years)" % (
site.name, end.year - start.year
))
sys.stderr.write("\n")
exclude = True
if exclude:
continue
include_sites.append(site)
for auth in site.sids.keys():
seen_auths.add(auth)
# Sort sites by longitude
include_sites = sorted(include_sites, key=lambda s: s.longitude)
seen_auths = sorted(seen_auths)
def get_val(site, field):
if hasattr(site, field):
return getattr(site, field)
else:
return site.sids.get(field, "")
def header(field):
vals = [get_val(site, field) for site in include_sites]
sys.stdout.write(",".join([field + ":"] + map(str, vals)))
sys.stdout.write("\n")
header("name")
for auth in seen_auths:
header(auth)
header("latitude")
header("longitude")
for year in range(syear, eyear + 1):
load_year_data(basin, elem, year, include_sites)