def test_strahler(self):
self.assertModule(
"v.stream.order",
input="stream_network",
points="stream_network_outlets",
output="stream_network_order_test_strahler",
threshold=25,
order=["strahler"],
overwrite=True,
verbose=True,
)
# Check the strahler value
v = VectorTopo(name="stream_network_order_test_strahler", mapset="")
v.open(mode="r")
self.assertTrue(v.exist(), True)
self.assertEqual(v.num_primitive_of("line"), 101)
# feature 4
self.assertEqual(v.read(4).attrs.cat, 41)
self.assertEqual(v.read(4).attrs["outlet_cat"], 1)
self.assertEqual(v.read(4).attrs["network"], 1)
self.assertEqual(v.read(4).attrs["reversed"], 0)
self.assertEqual(v.read(4).attrs["strahler"], 4)
v.close()
def _get_vector_features_as_wkb_list(lock, conn, data):
"""Return vector layer features as wkb list
supported feature types:
point, centroid, line, boundary, area
:param lock: A multiprocessing.Lock instance
:param conn: A multiprocessing.Pipe instance used to send True or False
:param data: The list of data entries [function_id,name,mapset,extent,
feature_type, field]
"""
wkb_list = None
try:
name = data[1]
mapset = data[2]
extent = data[3]
feature_type = data[4]
field = data[5]
bbox = None
mapset = utils.get_mapset_vector(name, mapset)
if not mapset:
raise ValueError("Unable to find vector map <%s>" % (name))
layer = VectorTopo(name, mapset)
if layer.exist() is True:
if extent is not None:
bbox = Bbox(
north=extent["north"],
south=extent["south"],
east=extent["east"],
west=extent["west"],
)
layer.open("r")
if feature_type.lower() == "area":
wkb_list = layer.areas_to_wkb_list(bbox=bbox, field=field)
else:
wkb_list = layer.features_to_wkb_list(
bbox=bbox, feature_type=feature_type, field=field)
layer.close()
finally:
# Send even if an exception was raised.
conn.send(wkb_list)
def _get_vector_table_as_dict(lock, conn, data):
"""Get the table of a vector map layer as dictionary
:param lock: A multiprocessing.Lock instance
:param conn: A multiprocessing.Pipe instance used to send True or False
:param data: The list of data entries [function_id, name, mapset, where]
"""
ret = None
try:
name = data[1]
mapset = data[2]
where = data[3]
mapset = utils.get_mapset_vector(name, mapset)
if not mapset:
raise ValueError("Unable to find vector map <%s>" % (name))
layer = VectorTopo(name, mapset)
if layer.exist() is True:
layer.open("r")
columns = None
table = None
if layer.table is not None:
columns = layer.table.columns
table = layer.table_to_dict(where=where)
layer.close()
ret = {}
ret["table"] = table
ret["columns"] = columns
finally:
# Send even if an exception was raised.
conn.send(ret)
def extendLine(map, map_out, maxlen=200, scale=0.5, debug=False, verbose=1):
#
# map=Input map name
# map_out=Output map with extensions
# maxlen=Max length in map units that line can be extended (def=200)
# scale=Maximum length of extension as proportion of original line, disabled if 0 (def=0.5)
# vlen=number of verticies to look back in calculating line end direction (def=1)
# Not sure if it is worth putting this in as parameter.
#
allowOverwrite = os.getenv('GRASS_OVERWRITE', '0') == '1'
grass.info("map={}, map_out={}, maxlen={}, scale={}, debug={}".format(
map, map_out, maxlen, scale, debug))
vlen = 1 # not sure if this is worth putting in as parameter
cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'parent', 'INTEGER'),
(u'dend', 'TEXT'), (u'orgx', 'DOUBLE PRECISION'),
(u'orgy', 'DOUBLE PRECISION'), (u'search_len', 'DOUBLE PRECISION'),
(u'search_az', 'DOUBLE PRECISION'), (u'best_xid', 'INTEGER'),
(u'near_x', 'DOUBLE PRECISION'), (u'near_y', 'DOUBLE PRECISION'),
(u'other_cat', 'INTEGER'), (u'xtype', 'TEXT'),
(u'x_len', 'DOUBLE PRECISION')]
extend = VectorTopo('extend')
if extend.exist():
extend.remove()
extend.open('w', tab_name='extend', tab_cols=cols)
#
# Go through input map, looking at each line and it's two nodes to find nodes
# with only a single line starting/ending there - i.e. a dangle.
# For each found, generate an extension line in the new map "extend"
#
inMap = VectorTopo(map)
inMap.open('r')
dangleCnt = 0
tickLen = len(inMap)
grass.info("Searching {} features for dangles".format(tickLen))
ticker = 0
grass.message("Percent complete...")
for ln in inMap:
ticker = (ticker + 1)
grass.percent(ticker, tickLen, 5)
if ln.gtype == 2: # Only process lines
for nd in ln.nodes():
if nd.nlines == 1: # We have a dangle
dangleCnt = dangleCnt + 1
vtx = min(len(ln) - 1, vlen)
if len([1 for _ in nd.lines(only_out=True)
]) == 1: # Dangle starting at node
dend = "head"
sx = ln[0].x
sy = ln[0].y
dx = sx - ln[vtx].x
dy = sy - ln[vtx].y
else: # Dangle ending at node
dend = "tail"
sx = ln[-1].x
sy = ln[-1].y
dx = sx - ln[-(vtx + 1)].x
dy = sy - ln[-(vtx + 1)].y
endaz = math.atan2(dy, dx)
if scale > 0:
extLen = min(ln.length() * scale, maxlen)
else:
extLen = maxlen
ex = extLen * math.cos(endaz) + sx
ey = extLen * math.sin(endaz) + sy
extLine = geo.Line([(sx, sy), (ex, ey)])
quiet = extend.write(extLine,
(ln.cat, dend, sx, sy, extLen, endaz,
0, 0, 0, 0, 'null', extLen))
grass.info(
"{} dangle nodes found, committing table extend".format(dangleCnt))
extend.table.conn.commit()
extend.close(build=True, release=True)
inMap.close()
#
# Create two tables where extensions intersect;
# 1. intersect with original lines
# 2. intersect with self - to extract intersects between extensions
#
# First the intersects with original lines
grass.info(
"Searching for intersects between potential extensions and original lines"
)
table_isectIn = Table('isectIn',
connection=sqlite3.connect(get_path(path)))
if table_isectIn.exist():
table_isectIn.drop(force=True)
run_command("v.distance",
flags='a',
overwrite=True,
quiet=True,
from_="extend",
from_type="line",
to=map,
to_type="line",
dmax="0",
upload="cat,dist,to_x,to_y",
column="near_cat,dist,nx,ny",
table="isectIn")
# Will have touched the dangle it comes from, so remove those touches
run_command(
"db.execute",
sql=
"DELETE FROM isectIn WHERE rowid IN (SELECT isectIn.rowid FROM isectIn INNER JOIN extend ON from_cat=cat WHERE near_cat=parent)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
run_command("db.execute",
sql="ALTER TABLE isectIn ADD ntype VARCHAR",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
run_command("db.execute",
sql="UPDATE isectIn SET ntype = 'orig' ",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
#
# Now second self intersect table
#
grass.info("Searching for intersects of potential extensions")
table_isectX = Table('isectX', connection=sqlite3.connect(get_path(path)))
if table_isectX.exist():
table_isectX.drop(force=True)
run_command("v.distance",
flags='a',
overwrite=True,
quiet=True,
from_="extend",
from_type="line",
to="extend",
to_type="line",
dmax="0",
upload="cat,dist,to_x,to_y",
column="near_cat,dist,nx,ny",
table="isectX")
# Obviously all extensions will intersect with themself, so remove those "intersects"
run_command("db.execute",
sql="DELETE FROM isectX WHERE from_cat = near_cat",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
run_command("db.execute",
sql="ALTER TABLE isectX ADD ntype VARCHAR",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
run_command("db.execute",
sql="UPDATE isectX SET ntype = 'ext' ",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
#
# Combine the two tables and add a few more attributes
#
run_command("db.execute",
sql="INSERT INTO isectIn SELECT * FROM isectX",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
cols_isectIn = Columns('isectIn',
connection=sqlite3.connect(get_path(path)))
cols_isectIn.add(['from_x'], ['DOUBLE PRECISION'])
cols_isectIn.add(['from_y'], ['DOUBLE PRECISION'])
cols_isectIn.add(['ext_len'], ['DOUBLE PRECISION'])
# Get starting coordinate at the end of the dangle
run_command(
"db.execute",
sql=
"UPDATE isectIn SET from_x = (SELECT extend.orgx FROM extend WHERE from_cat=extend.cat)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
run_command(
"db.execute",
sql=
"UPDATE isectIn SET from_y = (SELECT extend.orgy FROM extend WHERE from_cat=extend.cat)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
table_isectIn.conn.commit()
# For each intersect point, calculate the distance along extension line from end of dangle
# Would be nicer to do this in the database but SQLite dosen't support sqrt or exponents
grass.info(
"Calculating distances of intersects along potential extensions")
cur = table_isectIn.execute(
sql_code="SELECT rowid, from_x, from_y, nx, ny FROM isectIn")
for row in cur.fetchall():
rowid, fx, fy, nx, ny = row
x_len = math.sqrt((fx - nx)**2 + (fy - ny)**2)
sqlStr = "UPDATE isectIn SET ext_len={:.8f} WHERE rowid={:d}".format(
x_len, rowid)
table_isectIn.execute(sql_code=sqlStr)
grass.verbose("Ready to commit isectIn changes")
table_isectIn.conn.commit()
# Remove any zero distance from end of their dangle.
# This happens when another extension intersects exactly at that point
run_command("db.execute",
sql="DELETE FROM isectIn WHERE ext_len = 0.0",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
table_isectIn.conn.commit()
# Go through the extensions and find the intersect closest to each origin.
grass.info("Searching for closest intersect for each potential extension")
# db.execute sql="ALTER TABLE extend_t1 ADD COLUMN bst INTEGER"
# db.execute sql="ALTER TABLE extend_t1 ADD COLUMN nrx DOUBLE PRECISION"
# db.execute sql="ALTER TABLE extend_t1 ADD COLUMN nry DOUBLE PRECISION"
# db.execute sql="ALTER TABLE extend_t1 ADD COLUMN ocat TEXT"
# run_command("db.execute",
# sql = "INSERT OR REPLACE INTO extend_t1 (bst, nrx, nry, ocat) VALUES ((SELECT isectIn.rowid, ext_len, nx, ny, near_cat, ntype FROM isectIn WHERE from_cat=extend_t1.cat ORDER BY ext_len ASC LIMIT 1))",
# driver = "sqlite",
# database = "$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("CREATE index")
run_command("db.execute",
sql="CREATE INDEX idx_from_cat ON isectIn (from_cat)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("UPDATE best_xid")
run_command(
"db.execute",
sql=
"UPDATE extend SET best_xid = (SELECT isectIn.rowid FROM isectIn WHERE from_cat=extend.cat ORDER BY ext_len ASC LIMIT 1)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("UPDATE x_len")
run_command(
"db.execute",
sql=
"UPDATE extend SET x_len = (SELECT ext_len FROM isectIn WHERE from_cat=extend.cat ORDER BY ext_len ASC LIMIT 1)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("UPDATE near_x")
run_command(
"db.execute",
sql=
"UPDATE extend SET near_x = (SELECT nx FROM isectIn WHERE from_cat=extend.cat ORDER BY ext_len ASC LIMIT 1)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("UPDATE near_y")
run_command(
"db.execute",
sql=
"UPDATE extend SET near_y = (SELECT ny FROM isectIn WHERE from_cat=extend.cat ORDER BY ext_len ASC LIMIT 1)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("UPDATE other_cat")
run_command(
"db.execute",
sql=
"UPDATE extend SET other_cat = (SELECT near_cat FROM isectIn WHERE from_cat=extend.cat ORDER BY ext_len ASC LIMIT 1)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("UPDATE xtype")
run_command(
"db.execute",
sql=
"UPDATE extend SET xtype = (SELECT ntype FROM isectIn WHERE from_cat=extend.cat ORDER BY ext_len ASC LIMIT 1)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("DROP index")
run_command("db.execute",
sql="DROP INDEX idx_from_cat",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
grass.verbose("CREATE index on near_cat")
run_command("db.execute",
sql="CREATE INDEX idx_near_cat ON isectIn (near_cat)",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
quiet = table_isectIn.filters.select('rowid', 'ext_len', 'nx', 'ny',
'near_cat', 'ntype')
# quiet=table_isectIn.filters.order_by(['ext_len ASC'])
quiet = table_isectIn.filters.order_by('ext_len ASC')
quiet = table_isectIn.filters.limit(1)
table_extend = Table('extend', connection=sqlite3.connect(get_path(path)))
# Code below was relplaced by commands above untill memory problem can be sorted
# table_extend.filters.select('cat')
# cur=table_extend.execute()
# updateCnt = 0
# for row in cur.fetchall():
# cat, = row
# quiet=table_isectIn.filters.where('from_cat={:d}'.format(cat))
##SELECT rowid, ext_len, nx, ny, near_cat, ntype FROM isectIn WHERE from_cat=32734 ORDER BY ext_len ASC LIMIT 1
# x_sect=table_isectIn.execute().fetchone()
# if x_sect is not None:
# x_rowid, ext_len, nx, ny, other_cat, ntype = x_sect
# sqlStr="UPDATE extend SET best_xid={:d}, x_len={:.8f}, near_x={:.8f}, near_y={:.8f}, other_cat={:d}, xtype='{}' WHERE cat={:d}".format(x_rowid, ext_len, nx, ny, other_cat, ntype, cat)
# table_extend.execute(sql_code=sqlStr)
## Try periodic commit to avoide crash!
# updateCnt = (updateCnt + 1) % 10000
# if updateCnt == 0:
# table_extend.conn.commit()
grass.verbose("Ready to commit extend changes")
table_extend.conn.commit()
#
# There may be extensions that crossed, and that intersection chosen by one but
# not "recripricated" by the other.
# Need to remove those possibilities and allow the jilted extension to re-search.
#
grass.verbose("Deleting intersects already resolved")
run_command(
"db.execute",
sql=
"DELETE FROM isectIn WHERE rowid IN (SELECT isectIn.rowid FROM isectIn JOIN extend ON near_cat=cat WHERE ntype='ext' AND xtype!='null')", #"AND from_cat!=other_cat" no second chance!
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db")
table_isectIn.conn.commit()
grass.verbose("Deleting complete")
# To find the jilted - need a copy of extensions that have found an
# intersection (won't overwrite so drop first)
grass.verbose(
"Re-searching for mis-matched intersects between potential extensions")
table_imatch = Table('imatch', connection=sqlite3.connect(get_path(path)))
if table_imatch.exist():
table_imatch.drop(force=True)
wvar = "xtype!='null'"
run_command(
"db.copy",
overwrite=True,
quiet=True,
from_driver="sqlite",
from_database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db",
from_table="extend",
to_driver="sqlite",
to_database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db",
to_table="imatch",
where=wvar)
# Memory problems?
if gc.isenabled():
grass.verbose("Garbage collection enabled - forcing gc cycle")
gc.collect()
else:
grass.verbose("Garbage collection not enabled")
# Ensure tables are commited
table_extend.conn.commit()
table_imatch.conn.commit()
table_isectIn.conn.commit()
# Identify the jilted
sqlStr = "SELECT extend.cat FROM extend JOIN imatch ON extend.other_cat=imatch.cat WHERE extend.xtype='ext' and extend.cat!=imatch.other_cat"
cur = table_extend.execute(sql_code=sqlStr)
updateCnt = 0
for row in cur.fetchall():
cat, = row
grass.verbose("Reworking extend.cat={}".format(cat))
quiet = table_isectIn.filters.where('from_cat={:d}'.format(cat))
#print("SQL: {}".format(table_isectIn.filters.get_sql()))
x_sect = table_isectIn.execute().fetchone(
) ## Problem here under modules
if x_sect is None:
sqlStr = "UPDATE extend SET best_xid=0, x_len=search_len, near_x=0, near_y=0, other_cat=0, xtype='null' WHERE cat={:d}".format(
cat)
else:
x_rowid, ext_len, nx, ny, other_cat, ntype = x_sect
sqlStr = "UPDATE extend SET best_xid={:d}, x_len={:.8f}, near_x={:.8f}, near_y={:.8f}, other_cat={:d}, xtype='{}' WHERE cat={:d}".format(
x_rowid, ext_len, nx, ny, other_cat, ntype, cat)
table_extend.execute(sql_code=sqlStr)
## Try periodic commit to avoide crash!
updateCnt = (updateCnt + 1) % 100
if (updateCnt == 0): # or (cat == 750483):
grass.verbose(
"XXXXXXXXXXX Committing table_extend XXXXXXXXXXXXXXXXXXXXXX")
table_extend.conn.commit()
grass.verbose("Committing adjustments to table extend")
table_extend.conn.commit()
#
# For debugging, create a map with the chosen intersect points
#
if debug:
wvar = "xtype!='null' AND x_len!=0"
# print(wvar)
run_command(
"v.in.db",
overwrite=True,
quiet=True,
table="extend",
driver="sqlite",
database="$GISDBASE/$LOCATION_NAME/$MAPSET/sqlite/sqlite.db",
x="near_x",
y="near_y",
key="cat",
where=wvar,
output="chosen")
#
# Finally adjust the dangle lines in input map - use a copy (map_out) if requested
#
if map_out:
run_command("g.copy",
overwrite=allowOverwrite,
quiet=True,
vector=map + "," + map_out)
else: # Otherwise just modify the original dataset (map)
if allowOverwrite:
grass.warning("Modifying vector map ({})".format(map))
map_out = map
else:
grass.error(
"Use switch --o to modifying input vector map ({})".format(
map))
return 1
#
# Get info for lines that need extending
table_extend.filters.select(
'parent, dend, near_x, near_y, search_az, xtype')
table_extend.filters.where("xtype!='null'")
extLines = table_extend.execute().fetchall()
cat_mods = [ext[0] for ext in extLines]
tickLen = len(cat_mods)
grass.info("Extending {} dangles".format(tickLen))
ticker = 0
grass.message("Percent complete...")
# Open up the map_out copy (or the original) and work through looking for lines that need modifying
inMap = VectorTopo(map_out)
inMap.open('rw', tab_name=map_out)
for ln_idx in range(len(inMap)):
ln = inMap.read(ln_idx + 1)
if ln.gtype == 2: # Only process lines
while ln.cat in cat_mods: # Note: could be 'head' and 'tail'
ticker = (ticker + 1)
grass.percent(ticker, tickLen, 5)
cat_idx = cat_mods.index(ln.cat)
cat, dend, nx, ny, endaz, xtype = extLines.pop(cat_idx)
dump = cat_mods.pop(cat_idx)
if xtype == 'orig': # Overshoot by 0.1 as break lines is unreliable
nx = nx + 0.1 * math.cos(endaz)
ny = ny + 0.1 * math.sin(endaz)
newEnd = geo.Point(x=nx, y=ny, z=None)
if dend == 'head':
ln.insert(0, newEnd)
else: # 'tail'
ln.append(newEnd)
quiet = inMap.rewrite(ln_idx + 1, ln)
else:
quite = inMap.delete(ln_idx + 1)
## Try periodic commit and garbage collection to avoide crash!
if (ln_idx % 1000) == 0:
# inMap.table.conn.commit() - no such thing - Why??
if gc.isenabled():
quiet = gc.collect()
inMap.close(build=True, release=True)
grass.message("v.extendlines completing")
#
# Clean up temporary tables and maps
#
if not debug:
table_isectIn.drop(force=True)
table_isectX.drop(force=True)
table_imatch.drop(force=True)
extend.remove()
chosen = VectorTopo('chosen')
if chosen.exist():
chosen.remove()
return 0
def main():
from dateutil.parser import parse
try:
from pygbif import occurrences
from pygbif import species
except ImportError:
grass.fatal(
_("Cannot import pygbif (https://github.com/sckott/pygbif)"
" library."
" Please install it (pip install pygbif)"
" or ensure that it is on path"
" (use PYTHONPATH variable)."))
# Parse input options
output = options["output"]
mask = options["mask"]
species_maps = flags["i"]
no_region_limit = flags["r"]
no_topo = flags["b"]
print_species = flags["p"]
print_species_table = flags["t"]
print_species_shell = flags["g"]
print_occ_number = flags["o"]
allow_no_geom = flags["n"]
hasGeoIssue = flags["s"]
taxa_list = options["taxa"].split(",")
institutionCode = options["institutioncode"]
basisofrecord = options["basisofrecord"]
recordedby = options["recordedby"].split(",")
date_from = options["date_from"]
date_to = options["date_to"]
country = options["country"]
continent = options["continent"]
rank = options["rank"]
# Define static variable
# Initialize cat
cat = 0
# Number of occurrences to fetch in one request
chunk_size = 300
# lat/lon proj string
latlon_crs = [
"+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0.000,0.000,0.000",
"+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0,0,0,0,0,0,0",
"+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0.000,0.000,0.000 +type=crs",
]
# List attributes available in Darwin Core
# not all attributes are returned in each request
# to avoid key errors when accessing the dictionary returned by pygbif
# presence of DWC keys in the returned dictionary is checked using this list
# The number of keys in this list has to be equal to the number of columns
# in the attribute table and the attributes written for each occurrence
dwc_keys = [
"key",
"taxonRank",
"taxonKey",
"taxonID",
"scientificName",
"species",
"speciesKey",
"genericName",
"genus",
"genusKey",
"family",
"familyKey",
"order",
"orderKey",
"class",
"classKey",
"phylum",
"phylumKey",
"kingdom",
"kingdomKey",
"eventDate",
"verbatimEventDate",
"startDayOfYear",
"endDayOfYear",
"year",
"month",
"day",
"occurrenceID",
"occurrenceStatus",
"occurrenceRemarks",
"Habitat",
"basisOfRecord",
"preparations",
"sex",
"type",
"locality",
"verbatimLocality",
"decimalLongitude",
"decimalLatitude",
"coordinateUncertaintyInMeters",
"geodeticDatum",
"higerGeography",
"continent",
"country",
"countryCode",
"stateProvince",
"gbifID",
"protocol",
"identifier",
"recordedBy",
"identificationID",
"identifiers",
"dateIdentified",
"modified",
"institutionCode",
"lastInterpreted",
"lastParsed",
"references",
"relations",
"catalogNumber",
"occurrenceDetails",
"datasetKey",
"datasetName",
"collectionCode",
"rights",
"rightsHolder",
"license",
"publishingOrgKey",
"publishingCountry",
"lastCrawled",
"specificEpithet",
"facts",
"issues",
"extensions",
"language",
]
# Deinfe columns for attribute table
cols = [
("cat", "INTEGER PRIMARY KEY"),
("g_search", "varchar(100)"),
("g_key", "integer"),
("g_taxonrank", "varchar(50)"),
("g_taxonkey", "integer"),
("g_taxonid", "varchar(50)"),
("g_scientificname", "varchar(255)"),
("g_species", "varchar(255)"),
("g_specieskey", "integer"),
("g_genericname", "varchar(255)"),
("g_genus", "varchar(50)"),
("g_genuskey", "integer"),
("g_family", "varchar(50)"),
("g_familykey", "integer"),
("g_order", "varchar(50)"),
("g_orderkey", "integer"),
("g_class", "varchar(50)"),
("g_classkey", "integer"),
("g_phylum", "varchar(50)"),
("g_phylumkey", "integer"),
("g_kingdom", "varchar(50)"),
("g_kingdomkey", "integer"),
("g_eventdate", "text"),
("g_verbatimeventdate", "varchar(50)"),
("g_startDayOfYear", "integer"),
("g_endDayOfYear", "integer"),
("g_year", "integer"),
("g_month", "integer"),
("g_day", "integer"),
("g_occurrenceid", "varchar(255)"),
("g_occurrenceStatus", "varchar(50)"),
("g_occurrenceRemarks", "varchar(50)"),
("g_Habitat", "varchar(50)"),
("g_basisofrecord", "varchar(50)"),
("g_preparations", "varchar(50)"),
("g_sex", "varchar(50)"),
("g_type", "varchar(50)"),
("g_locality", "varchar(255)"),
("g_verbatimlocality", "varchar(255)"),
("g_decimallongitude", "double precision"),
("g_decimallatitude", "double precision"),
("g_coordinateUncertaintyInMeters", "double precision"),
("g_geodeticdatum", "varchar(50)"),
("g_higerGeography", "varchar(255)"),
("g_continent", "varchar(50)"),
("g_country", "varchar(50)"),
("g_countryCode", "varchar(50)"),
("g_stateProvince", "varchar(50)"),
("g_gbifid", "varchar(255)"),
("g_protocol", "varchar(255)"),
("g_identifier", "varchar(50)"),
("g_recordedby", "varchar(255)"),
("g_identificationid", "varchar(255)"),
("g_identifiers", "text"),
("g_dateidentified", "text"),
("g_modified", "text"),
("g_institutioncode", "varchar(50)"),
("g_lastinterpreted", "text"),
("g_lastparsed", "text"),
("g_references", "varchar(255)"),
("g_relations", "text"),
("g_catalognumber", "varchar(50)"),
("g_occurrencedetails", "text"),
("g_datasetkey", "varchar(50)"),
("g_datasetname", "varchar(255)"),
("g_collectioncode", "varchar(50)"),
("g_rights", "varchar(255)"),
("g_rightsholder", "varchar(255)"),
("g_license", "varchar(50)"),
("g_publishingorgkey", "varchar(50)"),
("g_publishingcountry", "varchar(50)"),
("g_lastcrawled", "text"),
("g_specificepithet", "varchar(50)"),
("g_facts", "text"),
("g_issues", "text"),
("g_extensions", "text"),
("g_language", "varchar(50)"),
]
# maybe no longer required in Python3
set_output_encoding()
# Set temporal filter if requested by user
# Initialize eventDate filter
eventDate = None
# Check if date from is compatible (ISO compliant)
if date_from:
try:
parse(date_from)
except:
grass.fatal("Invalid invalid start date provided")
if date_from and not date_to:
eventDate = "{}".format(date_from)
# Check if date to is compatible (ISO compliant)
if date_to:
try:
parse(date_to)
except:
grass.fatal("Invalid invalid end date provided")
# Check if date to is after date_from
if parse(date_from) < parse(date_to):
eventDate = "{},{}".format(date_from, date_to)
else:
grass.fatal(
"Invalid date range: End date has to be after start date!")
# Set filter on basisOfRecord if requested by user
if basisofrecord == "ALL":
basisOfRecord = None
else:
basisOfRecord = basisofrecord
# Allow also occurrences with spatial issues if requested by user
hasGeospatialIssue = False
if hasGeoIssue:
hasGeospatialIssue = True
# Allow also occurrences without coordinates if requested by user
hasCoordinate = True
if allow_no_geom:
hasCoordinate = False
# Set reprojection parameters
# Set target projection of current LOCATION
proj_info = grass.parse_command("g.proj", flags="g")
target_crs = grass.read_command("g.proj", flags="fj").rstrip()
target = osr.SpatialReference()
# Prefer EPSG CRS definitions
if proj_info["epsg"]:
target.ImportFromEPSG(int(proj_info["epsg"]))
else:
target.ImportFromProj4(target_crs)
# GDAL >= 3 swaps x and y axis, see: github.com/gdal/issues/1546
if int(gdal_version[0]) >= 3:
target.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
if target_crs == "XY location (unprojected)":
grass.fatal("Sorry, XY locations are not supported!")
# Set source projection from GBIF
source = osr.SpatialReference()
source.ImportFromEPSG(4326)
# GDAL >= 3 swaps x and y axis, see: github.com/gdal/issues/1546
if int(gdal_version[0]) >= 3:
source.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
if target_crs not in latlon_crs:
transform = osr.CoordinateTransformation(source, target)
reverse_transform = osr.CoordinateTransformation(target, source)
# Generate WKT polygon to use for spatial filtering if requested
if mask:
if len(mask.split("@")) == 2:
m = VectorTopo(mask.split("@")[0], mapset=mask.split("@")[1])
else:
m = VectorTopo(mask)
if not m.exist():
grass.fatal("Could not find vector map <{}>".format(mask))
m.open("r")
if not m.is_open():
grass.fatal("Could not open vector map <{}>".format(mask))
# Use map Bbox as spatial filter if map contains <> 1 area
if m.number_of("areas") == 1:
region_pol = [area.to_wkt() for area in m.viter("areas")][0]
else:
bbox = (str(m.bbox()).replace("Bbox(", "").replace(
" ", "").rstrip(")").split(","))
region_pol = "POLYGON (({0} {1}, {0} {3}, {2} {3}, {2} {1}, {0} {1}))".format(
bbox[2], bbox[0], bbox[3], bbox[1])
m.close()
else:
# Do not limit import spatially if LOCATION is able to take global data
if no_region_limit:
if target_crs not in latlon_crs:
grass.fatal("Import of data from outside the current region is"
"only supported in a WGS84 location!")
region_pol = None
else:
# Limit import spatially to current region
# if LOCATION is !NOT! able to take global data
# to avoid pprojection ERRORS
region = grass.parse_command("g.region", flags="g")
region_pol = "POLYGON (({0} {1},{0} {3},{2} {3},{2} {1},{0} {1}))".format(
region["e"], region["n"], region["w"], region["s"])
# Do not reproject in latlon LOCATIONS
if target_crs not in latlon_crs:
pol = ogr.CreateGeometryFromWkt(region_pol)
pol.Transform(reverse_transform)
pol = pol.ExportToWkt()
else:
pol = region_pol
# Create output map if not output maps for each species are requested
if (not species_maps and not print_species and not print_species_shell
and not print_occ_number and not print_species_table):
mapname = output
new = Vector(mapname)
new.open("w", tab_name=mapname, tab_cols=cols)
cat = 1
# Import data for each species
for s in taxa_list:
# Get the taxon key if not the taxon key is provided as input
try:
key = int(s)
except:
try:
species_match = species.name_backbone(s,
rank=rank,
strict=False,
verbose=True)
key = species_match["usageKey"]
except:
grass.error(
"Data request for taxon {} failed. Are you online?".format(
s))
continue
# Return matching taxon and alternatives and exit
if print_species:
print("Matching taxon for {} is:".format(s))
print("{} {}".format(species_match["scientificName"],
species_match["status"]))
if "alternatives" in list(species_match.keys()):
print("Alternative matches might be: {}".format(s))
for m in species_match["alternatives"]:
print("{} {}".format(m["scientificName"], m["status"]))
else:
print("No alternatives found for the given taxon")
continue
if print_species_shell:
print("match={}".format(species_match["scientificName"]))
if "alternatives" in list(species_match.keys()):
alternatives = []
for m in species_match["alternatives"]:
alternatives.append(m["scientificName"])
print("alternatives={}".format(",".join(alternatives)))
continue
if print_species_table:
if "alternatives" in list(species_match.keys()):
if len(species_match["alternatives"]) == 0:
print("{0}|{1}|{2}|".format(
s, key, species_match["scientificName"]))
else:
alternatives = []
for m in species_match["alternatives"]:
alternatives.append(m["scientificName"])
print("{0}|{1}|{2}|{3}".format(
s,
key,
species_match["scientificName"],
",".join(alternatives),
))
continue
try:
returns_n = occurrences.search(
taxonKey=key,
hasGeospatialIssue=hasGeospatialIssue,
hasCoordinate=hasCoordinate,
institutionCode=institutionCode,
basisOfRecord=basisOfRecord,
recordedBy=recordedby,
eventDate=eventDate,
continent=continent,
country=country,
geometry=pol,
limit=1,
)["count"]
except:
grass.error(
"Data request for taxon {} faild. Are you online?".format(s))
returns_n = 0
# Exit if search does not give a return
# Print only number of returns for the given search and exit
if print_occ_number:
print("Found {0} occurrences for taxon {1}...".format(
returns_n, s))
continue
elif returns_n <= 0:
grass.warning(
"No occurrences for current search for taxon {0}...".format(s))
continue
elif returns_n >= 200000:
grass.warning(
"Your search for {1} returns {0} records.\n"
"Unfortunately, the GBIF search API is limited to 200,000 records per request.\n"
"The download will be incomplete. Please consider to split up your search."
.format(returns_n, s))
# Get the number of chunks to download
chunks = int(math.ceil(returns_n / float(chunk_size)))
grass.verbose("Downloading {0} occurrences for taxon {1}...".format(
returns_n, s))
# Create a map for each species if requested using map name as suffix
if species_maps:
mapname = "{}_{}".format(s.replace(" ", "_"), output)
new = Vector(mapname)
new.open("w", tab_name=mapname, tab_cols=cols)
cat = 0
# Download the data from GBIF
for c in range(chunks):
# Define offset
offset = c * chunk_size
# Adjust chunk_size to the hard limit of 200,000 records in GBIF API
# if necessary
if offset + chunk_size >= 200000:
chunk_size = 200000 - offset
# Get the returns for the next chunk
returns = occurrences.search(
taxonKey=key,
hasGeospatialIssue=hasGeospatialIssue,
hasCoordinate=hasCoordinate,
institutionCode=institutionCode,
basisOfRecord=basisOfRecord,
recordedBy=recordedby,
eventDate=eventDate,
continent=continent,
country=country,
geometry=pol,
limit=chunk_size,
offset=offset,
)
# Write the returned data to map and attribute table
for res in returns["results"]:
if target_crs not in latlon_crs:
point = ogr.CreateGeometryFromWkt("POINT ({} {})".format(
res["decimalLongitude"], res["decimalLatitude"]))
point.Transform(transform)
x = point.GetX()
y = point.GetY()
else:
x = res["decimalLongitude"]
y = res["decimalLatitude"]
point = Point(x, y)
for k in dwc_keys:
if k not in list(res.keys()):
res.update({k: None})
cat = cat + 1
new.write(
point,
cat=cat,
attrs=(
"{}".format(s),
res["key"],
res["taxonRank"],
res["taxonKey"],
res["taxonID"],
res["scientificName"],
res["species"],
res["speciesKey"],
res["genericName"],
res["genus"],
res["genusKey"],
res["family"],
res["familyKey"],
res["order"],
res["orderKey"],
res["class"],
res["classKey"],
res["phylum"],
res["phylumKey"],
res["kingdom"],
res["kingdomKey"],
"{}".format(res["eventDate"])
if res["eventDate"] else None,
"{}".format(res["verbatimEventDate"])
if res["verbatimEventDate"] else None,
res["startDayOfYear"],
res["endDayOfYear"],
res["year"],
res["month"],
res["day"],
res["occurrenceID"],
res["occurrenceStatus"],
res["occurrenceRemarks"],
res["Habitat"],
res["basisOfRecord"],
res["preparations"],
res["sex"],
res["type"],
res["locality"],
res["verbatimLocality"],
res["decimalLongitude"],
res["decimalLatitude"],
res["coordinateUncertaintyInMeters"],
res["geodeticDatum"],
res["higerGeography"],
res["continent"],
res["country"],
res["countryCode"],
res["stateProvince"],
res["gbifID"],
res["protocol"],
res["identifier"],
res["recordedBy"],
res["identificationID"],
",".join(res["identifiers"]),
"{}".format(res["dateIdentified"])
if res["dateIdentified"] else None,
"{}".format(res["modified"])
if res["modified"] else None,
res["institutionCode"],
"{}".format(res["lastInterpreted"])
if res["lastInterpreted"] else None,
"{}".format(res["lastParsed"])
if res["lastParsed"] else None,
res["references"],
",".join(res["relations"]),
res["catalogNumber"],
"{}".format(res["occurrenceDetails"])
if res["occurrenceDetails"] else None,
res["datasetKey"],
res["datasetName"],
res["collectionCode"],
res["rights"],
res["rightsHolder"],
res["license"],
res["publishingOrgKey"],
res["publishingCountry"],
"{}".format(res["lastCrawled"])
if res["lastCrawled"] else None,
res["specificEpithet"],
",".join(res["facts"]),
",".join(res["issues"]),
",".join(res["extensions"]),
res["language"],
),
)
cat = cat + 1
# Close the current map if a map for each species is requested
if species_maps:
new.table.conn.commit()
new.close()
if not no_topo:
grass.run_command("v.build", map=mapname, option="build")
# Write history to map
grass.vector_history(mapname)
# Close the output map if not a map for each species is requested
if (not species_maps and not print_species and not print_species_shell
and not print_occ_number and not print_species_table):
new.table.conn.commit()
new.close()
if not no_topo:
grass.run_command("v.build", map=mapname, option="build")
# Write history to map
grass.vector_history(mapname)
def main():
in_vector = options["input"].split("@")[0]
if len(options["input"].split("@")) > 1:
in_mapset = options["input"].split("@")[1]
else:
in_mapset = None
raster_maps = options["raster"].split(
",") # raster file(s) to extract from
output = options["output"]
methods = tuple(options["methods"].split(","))
percentile = (None if options["percentile"] == "" else map(
float, options["percentile"].split(",")))
column_prefix = tuple(options["column_prefix"].split(","))
buffers = options["buffers"].split(",")
types = options["type"].split(",")
layer = options["layer"]
sep = options["separator"]
update = flags["u"]
tabulate = flags["t"]
percent = flags["p"]
remove = flags["r"]
use_label = flags["l"]
empty_buffer_warning = (
"No data in raster map {} within buffer {} around geometry {}")
# Do checks using pygrass
for rmap in raster_maps:
r_map = RasterAbstractBase(rmap)
if not r_map.exist():
grass.fatal("Could not find raster map {}.".format(rmap))
user_mask = False
m_map = RasterAbstractBase("MASK", Mapset().name)
if m_map.exist():
grass.warning("Current MASK is temporarily renamed.")
user_mask = True
unset_mask()
invect = VectorTopo(in_vector)
if not invect.exist():
grass.fatal("Vector file {} does not exist".format(in_vector))
if output:
if output == "-":
out = None
else:
out = open(output, "w")
# Check if input map is in current mapset (and thus editable)
if in_mapset and unicode(in_mapset) != unicode(Mapset()):
grass.fatal(
"Input vector map is not in current mapset and cannot be modified. \
Please consider copying it to current mapset.".format(
output))
buffers = []
for buf in options["buffers"].split(","):
try:
b = float(buf)
if b.is_integer():
buffers.append(int(b))
else:
buffers.append(b)
except:
grass.fatal("")
if b < 0:
grass.fatal("Negative buffer distance not supported!")
### Define column types depenting on statistic, map type and
### DB backend (SQLite supports only double and not real)
# int: statistic produces allways integer precision
# double: statistic produces allways floating point precision
# map_type: precision f statistic depends on map type
int_dict = {
"number": (0, "int", "n"),
"number_null": (1, "int", "null_cells"),
"minimum": (3, "map_type", "min"),
"maximum": (4, "map_type", "max"),
"range": (5, "map_type", "range"),
"average": (6, "double", "mean"),
"average_abs": (7, "double", "mean_of_abs"),
"stddev": (8, "double", "stddev"),
"variance": (9, "double", "variance"),
"coeff_var": (10, "double", "coeff_var"),
"sum": (11, "map_type", "sum"),
"first_quartile": (12, "map_type", "first_quartile"),
"median": (13, "map_type", "median"),
"third_quartile": (14, "map_type", "third_quartile"),
"percentile": (15, "map_type", "percentile"),
}
if len(raster_maps) != len(column_prefix):
grass.fatal(
"Number of maps and number of column prefixes has to be equal!")
# Generate list of required column names and types
col_names = []
valid_labels = []
col_types = []
for p in column_prefix:
rmaptype, val_lab, rcats = raster_type(
raster_maps[column_prefix.index(p)], tabulate, use_label)
valid_labels.append(val_lab)
for b in buffers:
b_str = str(b).replace(".", "_")
if tabulate:
if rmaptype == "double precision":
grass.fatal(
"{} has floating point precision. Can only tabulate integer maps"
.format(raster_maps[column_prefix.index(p)]))
col_names.append("{}_{}_b{}".format(p, "ncats", b_str))
col_types.append("int")
col_names.append("{}_{}_b{}".format(p, "mode", b_str))
col_types.append("int")
col_names.append("{}_{}_b{}".format(p, "null", b_str))
col_types.append("double precision")
col_names.append("{}_{}_b{}".format(p, "area_tot", b_str))
col_types.append("double precision")
for rcat in rcats:
if use_label and valid_labels:
rcat = rcat[0].replace(" ", "_")
else:
rcat = rcat[1]
col_names.append("{}_{}_b{}".format(p, rcat, b_str))
col_types.append("double precision")
else:
for m in methods:
col_names.append("{}_{}_b{}".format(
p, int_dict[m][2], b_str))
col_types.append(rmaptype if int_dict[m][1] ==
"map_type" else int_dict[m][1])
if percentile:
for perc in percentile:
col_names.append("{}_percentile_{}_b{}".format(
p,
int(perc) if (perc).is_integer() else perc, b_str))
col_types.append(rmaptype if int_dict[m][1] ==
"map_type" else int_dict[m][1])
# Open input vector map
in_vect = VectorTopo(in_vector, layer=layer)
in_vect.open(mode="r")
# Get name for temporary map
global TMP_MAPS
TMP_MAPS.append(tmp_map)
# Setup stats collectors
if tabulate:
# Collector for raster category statistics
stats = Module("r.stats", run_=False, stdout_=PIPE)
stats.inputs.sort = "desc"
stats.inputs.null_value = "null"
stats.flags.quiet = True
stats.flags.l = True
if percent:
stats.flags.p = True
stats.flags.n = True
else:
stats.flags.a = True
else:
# Collector for univariat statistics
univar = Module("r.univar", run_=False, stdout_=PIPE)
univar.inputs.separator = sep
univar.flags.g = True
univar.flags.quiet = True
# Add extended statistics if requested
if set(methods).intersection(
set(["first_quartile", "median", "third_quartile"])):
univar.flags.e = True
if percentile is not None:
univar.flags.e = True
univar.inputs.percentile = percentile
# Check if attribute table exists
if not output:
if not in_vect.table:
grass.fatal(
"No attribute table found for vector map {}".format(in_vect))
# Modify table as needed
tab = in_vect.table
tab_name = tab.name
tab_cols = tab.columns
# Add required columns
existing_cols = list(set(tab_cols.names()).intersection(col_names))
if len(existing_cols) > 0:
if not update:
in_vect.close()
grass.fatal(
"Column(s) {} already exist! Please use the u-flag \
if you want to update values in those columns".
format(",".join(existing_cols)))
else:
grass.warning("Column(s) {} already exist!".format(
",".join(existing_cols)))
for e in existing_cols:
idx = col_names.index(e)
del col_names[idx]
del col_types[idx]
tab_cols.add(col_names, col_types)
conn = tab.conn
cur = conn.cursor()
sql_str_start = "UPDATE {} SET ".format(tab_name)
elif output == "-":
print("cat{0}raster_map{0}buffer{0}statistic{0}value".format(sep))
else:
out.write("cat{0}raster_map{0}buffer{0}statistic{0}value{1}".format(
sep, os.linesep))
# Get computational region
grass.use_temp_region()
r = Region()
r.read()
# Adjust region extent to buffer around geometry
# reg = deepcopy(r)
# Create iterator for geometries of all selected types
geoms = chain()
geoms_n = 0
n_geom = 1
for geom_type in types:
geoms_n += in_vect.number_of(geom_type)
if in_vect.number_of(geom_type) > 0:
geoms = chain(in_vect.viter(geom_type))
# Loop over geometries
for geom in geoms:
# Get cat
cat = geom.cat
# Add where clause to UPDATE statement
sql_str_end = " WHERE cat = {};".format(cat)
# Loop over ser provided buffer distances
for buf in buffers:
b_str = str(buf).replace(".", "_")
# Buffer geometry
if buf <= 0:
buffer_geom = geom
else:
buffer_geom = geom.buffer(buf)
# Create temporary vector map with buffered geometry
tmp_vect = VectorTopo(tmp_map, quiet=True)
tmp_vect.open(mode="w")
tmp_vect.write(Boundary(points=buffer_geom[0].to_list()))
# , c_cats=int(cat), set_cats=True
if callable(buffer_geom[1]):
tmp_vect.write(Centroid(x=buffer_geom[1]().x,
y=buffer_geom[1]().y),
cat=int(cat))
else:
tmp_vect.write(Centroid(x=buffer_geom[1].x,
y=buffer_geom[1].y),
cat=int(cat))
#################################################
# How to silence VectorTopo???
#################################################
# Save current stdout
# original = sys.stdout
# f = open(os.devnull, 'w')
# with open('output.txt', 'w') as f:
# sys.stdout = io.BytesIO()
# sys.stdout.fileno() = os.devnull
# sys.stderr = f
# os.environ.update(dict(GRASS_VERBOSE='0'))
tmp_vect.close(build=False)
grass.run_command("v.build", map=tmp_map, quiet=True)
# os.environ.update(dict(GRASS_VERBOSE='1'))
# reg = Region()
# reg.read()
# r.from_vect(tmp_map)
r = align_current(r, buffer_geom[0].bbox())
r.write()
# Check if the following is needed
# needed specially with r.stats -p
# grass.run_command('g.region', vector=tmp_map, flags='a')
# Create a MASK from buffered geometry
if user_mask:
grass.run_command(
"v.to.rast",
input=tmp_map,
output=tmp_map,
use="val",
value=int(cat),
quiet=True,
)
mc_expression = (
"MASK=if(!isnull({0}) && !isnull({0}_MASK), {1}, null())".
format(tmp_map, cat))
grass.run_command("r.mapcalc",
expression=mc_expression,
quiet=True)
else:
grass.run_command(
"v.to.rast",
input=tmp_map,
output="MASK",
use="val",
value=int(cat),
quiet=True,
)
# reg.write()
updates = []
# Compute statistics for every raster map
for rm, rmap in enumerate(raster_maps):
# rmap = raster_maps[rm]
prefix = column_prefix[rm]
if tabulate:
# Get statistics on occurrence of raster categories within buffer
stats.inputs.input = rmap
stats.run()
t_stats = (stats.outputs["stdout"].value.rstrip(
os.linesep).replace(" ", " ").replace(
"no data", "no_data").replace(
" ",
"_b{} = ".format(b_str)).split(os.linesep))
if t_stats == [""]:
grass.warning(
empty_buffer_warning.format(rmap, buf, cat))
continue
if (t_stats[0].split(
"_b{} = ".format(b_str))[0].split("_")[-1] !=
"null"):
mode = (t_stats[0].split(
"_b{} = ".format(b_str))[0].split("_")[-1])
elif len(t_stats) == 1:
mode = "NULL"
else:
mode = (t_stats[1].split(
"_b{} = ".format(b_str))[0].split("_")[-1])
if not output:
updates.append("\t{}_{}_b{} = {}".format(
prefix, "ncats", b_str, len(t_stats)))
updates.append("\t{}_{}_b{} = {}".format(
prefix, "mode", b_str, mode))
area_tot = 0
for l in t_stats:
# check if raster maps has category or not
if len(l.split("=")) == 2:
updates.append("\t{}_{}".format(
prefix, l.rstrip("%")))
elif not l.startswith("null"):
vals = l.split("=")
updates.append("\t{}_{} = {}".format(
prefix,
vals[-2].strip()
if valid_labels[rm] else vals[0].strip(),
vals[-1].strip().rstrip("%"),
))
if not l.startswith("null"):
area_tot += float(
l.rstrip("%").split("= ")[-1])
if not percent:
updates.append("\t{}_{}_b{} = {}".format(
prefix, "area_tot", b_str, area_tot))
else:
out_str = "{1}{0}{2}{0}{3}{0}{4}{0}{5}{6}".format(
sep, cat, prefix, buf, "ncats", len(t_stats),
os.linesep)
out_str += "{1}{0}{2}{0}{3}{0}{4}{0}{5}{6}".format(
sep, cat, prefix, buf, "mode", mode, os.linesep)
area_tot = 0
for l in t_stats:
rcat = (l.split("= ")[1].rstrip(
"_b{} = ".format(b_str))
if valid_labels[rm] else l.split("_")[0])
area = l.split("= ")[-1]
out_str += "{1}{0}{2}{0}{3}{0}{4}{0}{5}{6}".format(
sep,
cat,
prefix,
buf,
"area {}".format(rcat),
area,
os.linesep,
)
if rcat != "null":
area_tot = area_tot + float(
l.rstrip("%").split("= ")[-1])
if not percent:
out_str += "{1}{0}{2}{0}{3}{0}{4}{0}{5}{6}".format(
sep,
cat,
prefix,
buf,
"area total",
area_tot,
os.linesep,
)
if output == "-":
print(out_str.rstrip(os.linesep))
else:
out.write(out_str)
else:
# Get univariate statistics within buffer
univar.inputs.map = rmap
univar.run()
u_stats = (univar.outputs["stdout"].value.rstrip(
os.linesep).replace(
"=", "_b{} = ".format(b_str)).split(os.linesep))
# Test if u_stats is empty and give warning
# Needs to be adjusted to number of requested stats?
if ((percentile and len(u_stats) < 14)
or (univar.flags.e and len(u_stats) < 13)
or len(u_stats) < 12):
grass.warning(
empty_buffer_warning.format(rmap, buf, cat))
break
# Extract statistics for selected methods
for m in methods:
if not output:
# Add to list of UPDATE statements
updates.append("\t{}_{}".format(
prefix,
u_stats[int_dict[m][0]] if is_number(
u_stats[int_dict[m][0]].split(" = ")[1])
else " = ".join([
u_stats[int_dict[m][0]].split(" = ")[0],
"NULL",
]),
))
else:
out_str = "{1}{0}{2}{0}{3}{0}{4}{0}{5}".format(
sep,
cat,
prefix,
buf,
m,
u_stats[int_dict[m][0]].split("= ")[1],
)
if output == "-":
print(out_str)
else:
out.write("{}{}".format(out_str, os.linesep))
if percentile:
perc_count = 0
for perc in percentile:
if not output:
updates.append(
"{}_percentile_{}_b{} = {}".format(
p,
int(perc) if
(perc).is_integer() else perc,
b_str,
u_stats[15 +
perc_count].split("= ")[1],
))
else:
out_str = "{1}{0}{2}{0}{3}{0}{4}{0}{5}".format(
sep,
cat,
prefix,
buf,
"percentile_{}".format(
int(perc) if (
perc).is_integer() else perc),
u_stats[15 + perc_count].split("= ")[1],
)
if output == "-":
print(out_str)
else:
out.write(out_str)
perc_count = perc_count + 1
if not output and len(updates) > 0:
cur.execute("{}{}{}".format(sql_str_start, ",\n".join(updates),
sql_str_end))
# Remove temporary maps
# , stderr=os.devnull, stdout_=os.devnull)
grass.run_command("g.remove",
flags="f",
type="raster",
name="MASK",
quiet=True)
grass.run_command("g.remove",
flags="f",
type="vector",
name=tmp_map,
quiet=True)
# Give progress information
grass.percent(n_geom, geoms_n, 1)
n_geom = n_geom + 1
if not output:
conn.commit()
# Close cursor and DB connection
if not output and not output == "-":
cur.close()
conn.close()
# Update history
grass.vector.vector_history(in_vector)
elif output != "-":
# write results to file
out.close()
if remove and not output:
dropcols = []
selectnum = "select count({}) from {}"
for i in col_names:
thisrow = grass.read_command("db.select",
flags="c",
sql=selectnum.format(i, in_vector))
if int(thisrow) == 0:
dropcols.append(i)
grass.debug("Columns to delete: {}".format(", ".join(dropcols)),
debug=2)
if dropcols:
grass.run_command("v.db.dropcolumn",
map=in_vector,
columns=dropcols)
def main():
in_vector = options['input'].split('@')[0]
if len(options['input'].split('@')) > 1:
in_mapset = options['input'].split('@')[1]
else:
in_mapset = None
raster_maps = options['raster'].split(',') # raster file(s) to extract from
output = options['output']
methods = tuple(options['methods'].split(','))
percentile = None if options['percentile'] == '' else map(float, options['percentile'].split(','))
column_prefix = tuple(options['column_prefix'].split(','))
buffers = options['buffers'].split(',')
types = options['type'].split(',')
layer = options['layer']
sep = options['separator']
update = flags['u']
tabulate = flags['t']
percent = flags['p']
remove = flags['r']
use_lable = False
empty_buffer_warning = 'No data in raster map {} within buffer {} around geometry {}'
# Do checks using pygrass
for rmap in raster_maps:
r_map = RasterAbstractBase(rmap)
if not r_map.exist():
grass.fatal('Could not find raster map {}.'.format(rmap))
user_mask = False
m_map = RasterAbstractBase('MASK', Mapset().name)
if m_map.exist():
grass.warning("Current MASK is temporarily renamed.")
user_mask = True
unset_mask()
invect = VectorTopo(in_vector)
if not invect.exist():
grass.fatal("Vector file {} does not exist".format(in_vector))
if output:
if output == '-':
out = None
else:
out = open(output, 'w')
# Check if input map is in current mapset (and thus editable)
if in_mapset and unicode(in_mapset) != unicode(Mapset()):
grass.fatal("Input vector map is not in current mapset and cannot be modified. \
Please consider copying it to current mapset.".format(output))
buffers = []
for buf in options['buffers'].split(','):
try:
b = float(buf)
if b.is_integer():
buffers.append(int(b))
else:
buffers.append(b)
except:
grass.fatal('')
if b < 0:
grass.fatal("Negative buffer distance not supported!")
### Define column types depenting on statistic, map type and
### DB backend (SQLite supports only double and not real)
# int: statistic produces allways integer precision
# double: statistic produces allways floating point precision
# map_type: precision f statistic depends on map type
int_dict = {'number': (0, 'int', 'n'),
'number_null': (1, 'int', 'null_cells'),
'minimum': (3, 'map_type', 'min'),
'maximum': (4, 'map_type', 'max'),
'range': (5, 'map_type', 'range'),
'average': (6, 'double', 'mean'),
'average_abs': (7, 'double', 'mean_of_abs'),
'stddev': (8, 'double', 'stddev'),
'variance': (9, 'double', 'variance'),
'coeff_var': (10, 'double', 'coeff_var'),
'sum': (11, 'map_type', 'sum'),
'first_quartile': (12, 'map_type', 'first_quartile'),
'median': (13, 'map_type', 'median'),
'third_quartile': (14, 'map_type', 'third_quartile'),
'percentile': (15, 'map_type', 'percentile')}
if len(raster_maps) != len(column_prefix):
grass.fatal('Number of maps and number of column prefixes has to be equal!')
# Generate list of required column names and types
col_names = []
col_types = []
for p in column_prefix:
rmaptype, rcats = raster_type(raster_maps[column_prefix.index(p)], tabulate, use_lable)
for b in buffers:
b_str = str(b).replace('.', '_')
if tabulate:
if rmaptype == 'double precision':
grass.fatal('{} has floating point precision. Can only tabulate integer maps'.format(raster_maps[column_prefix.index(p)]))
col_names.append('{}_{}_b{}'.format(p, 'ncats', b_str))
col_types.append('int')
col_names.append('{}_{}_b{}'.format(p, 'mode', b_str))
col_types.append('int')
col_names.append('{}_{}_b{}'.format(p, 'null', b_str))
col_types.append('double precision')
col_names.append('{}_{}_b{}'.format(p, 'area_tot', b_str))
col_types.append('double precision')
for rcat in rcats:
if use_lable:
rcat = rcat[1].replace(" ", "_")
else:
rcat = rcat[0]
col_names.append('{}_{}_b{}'.format(p, rcat, b_str))
col_types.append('double precision')
else:
for m in methods:
col_names.append('{}_{}_b{}'.format(p, int_dict[m][2], b_str))
col_types.append(rmaptype if int_dict[m][1] == 'map_type' else int_dict[m][1])
if percentile:
for perc in percentile:
col_names.append('{}_percentile_{}_b{}'.format(p,
int(perc) if (perc).is_integer() else perc,
b_str))
col_types.append(rmaptype if int_dict[m][1] == 'map_type' else int_dict[m][1])
# Open input vector map
in_vect = VectorTopo(in_vector, layer=layer)
in_vect.open(mode='r')
# Get name for temporary map
TMP_MAPS.append(tmp_map)
# Setup stats collectors
if tabulate:
# Collector for raster category statistics
stats = Module('r.stats', run_=False, stdout_=PIPE)
stats.inputs.sort = 'desc'
stats.inputs.null_value = 'null'
stats.flags.quiet = True
if percent:
stats.flags.p = True
stats.flags.n = True
else:
stats.flags.a = True
else:
# Collector for univariat statistics
univar = Module('r.univar', run_=False, stdout_=PIPE)
univar.inputs.separator = sep
univar.flags.g = True
univar.flags.quiet = True
# Add extended statistics if requested
if set(methods).intersection(set(['first_quartile',
'median', 'third_quartile'])):
univar.flags.e = True
if percentile is not None:
univar.flags.e = True
univar.inputs.percentile = percentile
# Check if attribute table exists
if not output:
if not in_vect.table:
grass.fatal('No attribute table found for vector map {}'.format(in_vect))
# Modify table as needed
tab = in_vect.table
tab_name = tab.name
tab_cols = tab.columns
# Add required columns
existing_cols = list(set(tab_cols.names()).intersection(col_names))
if len(existing_cols) > 0:
if not update:
grass.fatal('Column(s) {} already exist! Please use the u-flag \
if you want to update values in those columns'.format(','.join(existing_cols)))
else:
grass.warning('Column(s) {} already exist!'.format(','.join(existing_cols)))
for e in existing_cols:
idx = col_names.index(e)
del col_names[idx]
del col_types[idx]
tab_cols.add(col_names, col_types)
conn = tab.conn
cur = conn.cursor()
sql_str_start = 'UPDATE {} SET '.format(tab_name)
elif output == '-':
print('cat{0}raster_map{0}buffer{0}statistic{0}value'.format(sep))
else:
out.write('cat{0}raster_map{0}buffer{0}statistic{0}value{1}'.format(sep, os.linesep))
# Get computational region
grass.use_temp_region()
r = Region()
r.read()
# Adjust region extent to buffer around geometry
#reg = deepcopy(r)
# Create iterator for geometries of all selected types
geoms = chain()
geoms_n = 0
n_geom = 1
for geom_type in types:
geoms_n += in_vect.number_of(geom_type)
if in_vect.number_of(geom_type) > 0:
geoms = chain(in_vect.viter(geom_type))
# Loop over geometries
for geom in geoms:
# Get cat
cat = geom.cat
# Add where clause to UPDATE statement
sql_str_end = ' WHERE cat = {};'.format(cat)
# Loop over ser provided buffer distances
for buf in buffers:
b_str = str(buf).replace('.', '_')
# Buffer geometry
if buf <= 0:
buffer_geom = geom
else:
buffer_geom = geom.buffer(buf)
# Create temporary vector map with buffered geometry
tmp_vect = VectorTopo(tmp_map, quiet=True)
tmp_vect.open(mode='w')
#print(int(cat))
tmp_vect.write(Boundary(points=buffer_geom[0].to_list()))
# , c_cats=int(cat), set_cats=True
tmp_vect.write(Centroid(x=buffer_geom[1].x,
y=buffer_geom[1].y), cat=int(cat))
#################################################
# How to silence VectorTopo???
#################################################
# Save current stdout
#original = sys.stdout
#f = open(os.devnull, 'w')
#with open('output.txt', 'w') as f:
#sys.stdout = io.BytesIO()
#sys.stdout.fileno() = os.devnull
#sys.stderr = f
#os.environ.update(dict(GRASS_VERBOSE='0'))
tmp_vect.close(build=False)
grass.run_command('v.build', map=tmp_map, quiet=True)
#os.environ.update(dict(GRASS_VERBOSE='1'))
#reg = Region()
#reg.read()
#r.from_vect(tmp_map)
r = align_current(r, buffer_geom[0].bbox())
r.write()
# Check if the following is needed
# needed specially with r.stats -p
#grass.run_command('g.region', vector=tmp_map, flags='a')
# Create a MASK from buffered geometry
if user_mask:
grass.run_command('v.to.rast', input=tmp_map,
output=tmp_map, use='val',
value=int(cat), quiet=True)
mc_expression = "MASK=if(!isnull({0}) && !isnull({0}_MASK), {1}, null())".format(tmp_map, cat)
grass.run_command('r.mapcalc', expression=mc_expression, quiet=True)
else:
grass.run_command('v.to.rast', input=tmp_map,
output='MASK', use='val',
value=int(cat), quiet=True)
#reg.write()
updates = []
# Compute statistics for every raster map
for rm in range(len(raster_maps)):
rmap = raster_maps[rm]
prefix = column_prefix[rm]
if tabulate:
# Get statistics on occurrence of raster categories within buffer
stats.inputs.input = rmap
stats.run()
t_stats = stats.outputs['stdout'].value.rstrip(os.linesep).replace(' ', '_b{} = '.format(b_str)).split(os.linesep)
if t_stats[0].split('_b{} = '.format(b_str))[0].split('_')[-1] != 'null':
mode = t_stats[0].split('_b{} = '.format(b_str))[0].split('_')[-1]
elif len(t_stats) == 1:
mode = 'NULL'
else:
mode = t_stats[1].split('_b{} = '.format(b_str))[0].split('_')[-1]
if not output:
updates.append('\t{}_{}_b{} = {}'.format(prefix, 'ncats', b_str, len(t_stats)))
updates.append('\t{}_{}_b{} = {}'.format(prefix, 'mode', b_str, mode))
area_tot = 0
for l in t_stats:
updates.append('\t{}_{}'.format(prefix, l.rstrip('%')))
if l.split('_b{} ='.format(b_str))[0].split('_')[-1] != 'null':
area_tot = area_tot + float(l.rstrip('%').split('= ')[1])
if not percent:
updates.append('\t{}_{}_b{} = {}'.format(prefix, 'area_tot', b_str, area_tot))
else:
out_str = '{1}{0}{2}{0}{3}{0}{4}{0}{5}{6}'.format(sep, cat, prefix, buf, 'ncats', len(t_stats), os.linesep)
out_str += '{1}{0}{2}{0}{3}{0}{4}{0}{5}{6}'.format(sep, cat, prefix, buf, 'mode', mode, os.linesep)
area_tot = 0
if not t_stats[0]:
grass.warning(empty_buffer_warning.format(rmap, buf, cat))
continue
for l in t_stats:
rcat = l.split('_b{} ='.format(b_str))[0].split('_')[-1]
area = l.split('= ')[1]
out_str += '{1}{0}{2}{0}{3}{0}{4}{0}{5}{6}'.format(sep, cat, prefix, buf, 'area {}'.format(rcat), area, os.linesep)
if rcat != 'null':
area_tot = area_tot + float(l.rstrip('%').split('= ')[1])
out_str += '{1}{0}{2}{0}{3}{0}{4}{0}{5}{6}'.format(sep, cat, prefix, buf, 'area_tot', area_tot, os.linesep)
if output == '-':
print(out_str.rstrip(os.linesep))
else:
out.write(out_str)
else:
# Get univariate statistics within buffer
univar.inputs.map = rmap
univar.run()
u_stats = univar.outputs['stdout'].value.rstrip(os.linesep).replace('=', '_b{} = '.format(b_str)).split(os.linesep)
# Test if u_stats is empty and give warning
# Needs to be adjusted to number of requested stats?
if (percentile and len(u_stats) < 14) or (univar.flags.e and len(u_stats) < 13) or len(u_stats) < 12:
grass.warning(empty_buffer_warning.format(rmap, buf, cat))
break
# Extract statistics for selected methods
for m in methods:
if not output:
# Add to list of UPDATE statements
updates.append('\t{}_{}'.format(prefix,
u_stats[int_dict[m][0]]))
else:
out_str = '{1}{0}{2}{0}{3}{0}{4}{0}{5}'.format(sep, cat, prefix, buf, m, u_stats[int_dict[m][0]].split('= ')[1])
if output == '-':
print(out_str)
else:
out.write("{}{}".format(out_str, os.linesep))
if percentile:
perc_count = 0
for perc in percentile:
if not output:
updates.append('{}_percentile_{}_b{} = {}'.format(p,
int(perc) if (perc).is_integer() else perc,
b_str, u_stats[15+perc_count].split('= ')[1]))
else:
out_str = '{1}{0}{2}{0}{3}{0}{4}{0}{5}'.format(sep, cat, prefix, buf, 'percentile_{}'.format(int(perc) if (perc).is_integer() else perc), u_stats[15+perc_count].split('= ')[1])
if output == '-':
print(out_str)
else:
out.write(out_str)
perc_count = perc_count + 1
if not output and len(updates) > 0:
cur.execute('{}{}{}'.format(sql_str_start,
',\n'.join(updates),
sql_str_end))
# Remove temporary maps
#, stderr=os.devnull, stdout_=os.devnull)
grass.run_command('g.remove', flags='f', type='raster',
name='MASK', quiet=True)
grass.run_command('g.remove', flags='f', type='vector',
name=tmp_map, quiet=True)
# Give progress information
grass.percent(n_geom, geoms_n, 1)
n_geom = n_geom + 1
if not output:
conn.commit()
# Close cursor and DB connection
if not output and not output == "-":
cur.close()
conn.close()
# Update history
grass.vector.vector_history(in_vector)
elif output != "-":
# write results to file
out.close()
if remove:
dropcols = []
selectnum = 'select count({}) from {}'
for i in col_names:
thisrow = grass.read_command('db.select', flags='c',
sql=selectnum.format(i, in_vector))
if int(thisrow) == 0:
dropcols.append(i)
grass.debug("Columns to delete: {}".format(', '.join(dropcols)),
debug=2)
grass.run_command('v.db.dropcolumn', map=in_vector, columns=dropcols)
def main():
dem = options['dem']
TR = options['time'] #TODO Time of concentration
outlet = options['outlets']
outlets = outlet.split(',')
cleanTemporary = options['clean']
try:
TR = int(TR)
except:
print 'TR is not a number'
sys.exit()
if cleanTemporary != 'no':
grass.run_command(
'g.remove',
flags='f',
type='raster',
name=
'main_stream,basin,circle,drainage,horton,raster_streams,slope_drain_into'
)
grass.run_command('g.remove',
flags='f',
type='vector',
name='main_stream,nodes,outlet')
grass.run_command('g.remove',
type='vector',
pattern='main_stream*',
flags='f')
grass.use_temp_region()
#get region in order to estimate the threshold as 1/1000 of total cells
grass.run_command('g.region', raster=dem)
regione = grass.region()
thrshold = float(regione['cells']) / 300
#stream and drainage determination
grass.run_command('r.watershed',
elevation=dem,
threshold=700,
stream='raster_streams',
drainage='drainage',
overwrite=True,
flags='s')
#the radius is little more than the current resolution
radius = regione['nsres'] * 1.4
grass.run_command('r.circle',
output='circle',
coordinate=outlet,
max=radius,
overwrite=True) #%(str(outlets[0]),str(outlets[1]))
#get the distances and take the shortest distance
distances = grass.read_command('r.distance', map='circle,raster_streams')
list_dist = distances.split('\n')
list_dist.remove('')
list_tuple = []
for distance in list_dist:
dist = distance.split(':')
my_tupla = dist[0], dist[1], float(
dist[2]), dist[3], dist[4], dist[5], dist[6]
list_tuple.append(my_tupla)
tuple_orderedByDistance = sorted(list_tuple,
key=lambda distanza: distanza[2])
del (distances, list_tuple, list_dist)
#calculate the basin and read its statistics
outlet = tuple_orderedByDistance[0][-2:]
xoutlet = float(outlet[0])
youtlet = float(outlet[1])
grass.run_command('r.water.outlet',
input='drainage',
output='basin',
coordinates=str(xoutlet) + ',' + str(youtlet),
overwrite=True)
statistics = grass.read_command('r.univar', map=dem, zones='basin')
main_stat = statistics.splitlines()[-9:]
#order the stream network
grass.run_command('r.mask', raster='basin')
grass.run_command('r.stream.order',
stream_rast='raster_streams',
direction='drainage',
elevation=dem,
horton='horton',
overwrite=True)
stream_stat = grass.read_command('r.stream.stats',
stream_rast='horton',
direction='drainage',
elevation=dem,
flags='o')
network_statistics = stream_stat.split('\n')
network_statistics.remove('')
#get the max order
network_statistics[-1].split()
total_length = float(network_statistics[-1].split(',')[2])
area_basin = float(network_statistics[-1].split(',')[3])
#area_basin in km2
area_basin_Ha = area_basin * 100
mean_elev = float(main_stat[3].split(':')[-1])
min_elev = float(main_stat[0].split(':')[-1])
max_elev = float(main_stat[1].split(':')[-1])
deltaH = max_elev - min_elev
average_slope = float(network_statistics[-1].split(',')[4])
grass.run_command('r.mask', flags='r')
TcGiandotti = (4 * np.sqrt(area_basin) +
1.5 * total_length) / (0.8 * np.sqrt(mean_elev - min_elev))
TcKirpich = 0.945 * (total_length**3. / deltaH)**0.385
if area_basin_Ha > 1000: #TODO controlla i riferimenti
corrivazione = TcGiandotti
grass.info('using giandotti')
grass.info(str(TcGiandotti))
formula = 'Giandotti'
else:
formula = 'Kirpich'
corrivazione = TcKirpich
grass.info('using Kirpich')
grass.info(str(TcKirpich))
if corrivazione < 24:
aPar = 'a24@PERMANENT'
bPar = 'b24@PERMANENT'
kPar = 'k24@PERMANENT'
else:
aPar = 'a15@PERMANENT'
bPar = 'b15@PERMANENT'
kPar = 'k15@PERMANENT'
CNmap = 'CN@PERMANENT'
aStat = grass.read_command('r.univar', map=aPar, zones='basin')
aMain_stat = aStat.splitlines()[12].split(':')[-1]
aMain_stat = float(aMain_stat)
bStat = grass.read_command('r.univar', map=bPar, zones='basin')
bMain_stat = bStat.splitlines()[12].split(':')[-1]
bMain_stat = float(bMain_stat)
kStat = grass.read_command('r.univar', map=kPar, zones='basin')
kMain_stat = kStat.splitlines()[12].split(':')[-1]
kMain_stat = float(kMain_stat)
CNstat = grass.read_command('r.univar', map=CNmap, zones='basin')
CN = CNstat.splitlines()[12].split(':')[-1]
CN = float(CN)
g.message('area basin in km2: ')
print area_basin
print 'mean elev: '
print mean_elev - min_elev
print 'delta H:'
print deltaH
print 'total reach length: '
print total_length
print 'a mean:'
print aMain_stat
print '\n b mean: '
print bMain_stat
print '\n k mean: '
print kMain_stat
print 'CN mean:'
print CN
##### ------------------------- ##### modifica per verifca da togliere
# ~ corrivazione=3.
# ~ aMain_stat=32.5
# ~ bMain_stat=0.33
# ~ kMain_stat=0.42
# ~ CN=91.
# ~ area_basin=61.5
# ~ area_basin_Ha=area_basin*100
CN = 70.12 / 82.63 * CN
#CN = 78.6
#####--------------------------#####
f_K_T = 1 - kMain_stat * (0.45 + 0.799 * np.log(-np.log(1 - 1. / TR)))
print 'f(k,T): '
print f_K_T
h = f_K_T * aMain_stat * corrivazione**bMain_stat
print '\n h main:'
print h
X1 = 100 * corrivazione / (0.236 + 0.062 * corrivazione)
X2 = 0.003 * corrivazione + 0.0234
Pa = 100 - area_basin_Ha / (X1 + X2 * area_basin_Ha)
Ha = h * Pa / 100
S1 = (1000. / CN) - 10
Pn = (Ha - 5.08 * S1)**2 / (Ha + 20.32 * S1)
Qc = (1 / 360.) * Pn * area_basin_Ha / corrivazione
print 'discharge: '
print Qc
#print table.columns.types()
#[u'INTEGER', u'TEXT', u'integer', u'double precision']
'''
------------------------------
START CALCULATION OF LOCAL UPSTREAM SLOPE
------------------------------
'''
#offsets for moving windows
offsets = [
d for j in xrange(1, 1 + 1) for i in [j, -j]
for d in [(i, 0), (0, i), (i, i), (i, -i)]
]
#rename dtm as elevation for future calculation if not exist
if not VectorTopo('elevation').exist():
grass.run_command('g.rename', raster="%s,elevation" % dem)
elev_renamed = True
#define drainage direction
drainage_incoming = [2, 4, 3, 1, 6, 8, 7, 5]
drainage_outcoming = []
diag_dist = (regione['nsres']**2 + regione['ewres']**2)**0.5
# [(1, 0), (0, 1), (1, 1), (1, -1), (-1, 0), (0, -1), (-1, -1), (-1, 1),
cell_dists = [
regione['nsres'], regione['ewres'], diag_dist, diag_dist,
regione['nsres'], regione['ewres'], diag_dist, diag_dist
]
# define the calculation term
terms = [
"(drainage[%d,%d] == %d && not(isnull(raster_streams[0,0])) && not(isnull(raster_streams[%d,%d])) )"
% ((offsets[j] + tuple([drainage_incoming[j]]) + offsets[j]))
for j in range(len(drainage_incoming))
]
#define the operation expression
terms_calc = [
"(elevation[%d,%d] - elevation) * %s" % (offsets[j] + (terms[j], ))
for j in range(len(terms))
]
terms_calc_slope = [
"( (elevation[%d,%d] - elevation)/%10.4f ) * %s" %
(offsets[j] + (cell_dists[j], ) + (terms[j], ))
for j in range(len(terms))
]
expr = "num_cells_drain_into = (%s)" % " + ".join(terms)
expr1 = "elevation_percentile4 = if(isnull(raster_streams),null(),(%s))" % " + ".join(
terms)
expr2 = "elevdiff_drain_into = %s" % " + ".join(terms_calc)
expr3 = "slope_drain_into = %s" % " + ".join(terms_calc_slope)
# do the r.mapcalc calculation with the moving window
# exclude the num_cell_calculation_into
#grass.mapcalc( expr )
#print expr2
#grass.mapcalc( expr2 , overwrite=True)
#print expr3
#grass.mapcalc( expr3 , overwrite=True)
'''
------------------------------
START CALCULATION OF 2KM UPSTREAM SLOPE
------------------------------
'''
#create an outlet vector
new = VectorTopo('outlet')
COLS = [(u'cat', 'INTEGER PRIMARY KEY')]
new.open('w', tab_name='outlet', tab_cols=COLS)
new.write(
Point(xoutlet, youtlet),
cat=1,
)
new.table.conn.commit()
new.table.execute().fetchall()
new.close()
new = VectorTopo('output')
COLS = [(u'cat', 'INTEGER PRIMARY KEY'),
(u'discharge', u'double precision')]
if new.exist():
g.message('The vector exist: it will be named as')
new_name = 'output' + str(datetime.datetime.now().time())[:8].replace(
':', '_')
grass.info(new_name)
new = VectorTopo(new_name)
new.open('w', tab_name=new_name, tab_cols=COLS)
else:
new.open('w', tab_name='output', tab_cols=COLS)
new.write(Point(xoutlet, youtlet), (float(Qc), ))
new.table.conn.commit()
new.table.execute().fetchall()
new.close()
mean_elev_abs = mean_elev - min_elev
FirsPage(formula=formula,
xoutlet=xoutlet,
youtlet=youtlet,
Tc=corrivazione,
AreaBasin=area_basin_Ha,
ChLen=total_length * 1000.,
MeanElev=mean_elev_abs,
a=aMain_stat,
b=bMain_stat,
k=kMain_stat,
TR=TR,
f=f_K_T,
h=h,
h_ar=Ha,
x1=X1,
x2=X2,
Pa=Pa,
CN=CN,
S1=S1,
Pn=Pn,
Qc=Qc,
dropElev=deltaH)
#cleaning part
if elev_renamed:
grass.run_command('g.rename', raster='elevation,%s' % dem)
grass.del_temp_region()
grass.run_command('r.to.vect',
input='basin',
output='basin1',
type='area',
overwrite=True)
if cleanTemporary != 'no':
grass.run_command(
'g.remove',
flags='f',
type='raster',
name=
'main_stream,basin,circle,drainage,horton,raster_streams,slope_drain_into'
)
grass.run_command('g.remove',
flags='f',
type='vector',
name='main_stream,nodes,outlet')
grass.run_command('g.remove',
type='vector',
pattern='main_stream*',
flags='f')
def create_maps(parsed_obs, offering, secondsGranularity, resolution):
"""
Create raster maps representing offerings, observed props and procedures
:param parsed_obs: Observations for a given offering in geoJSON format
:param offering: A collection of sensors used to conveniently group them up
:param secondsGranularity: Granularity in seconds
:param resolution: 2D grid resolution for rasterization
"""
timestampPattern = '%Y-%m-%dT%H:%M:%S' # TODO: Timezone
startTime = options['event_time'].split('+')[0]
epochS = int(time.mktime(time.strptime(startTime, timestampPattern)))
endTime = options['event_time'].split('+')[1].split('/')[1]
epochE = int(time.mktime(time.strptime(endTime, timestampPattern)))
for key, observation in parsed_obs.iteritems():
print('Creating raster maps for offering '
'{}, observed property {}'.format(offering, key))
data = json.loads(observation)
cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'name', 'VARCHAR'),
(u'value', 'DOUBLE')]
geometries = dict()
intervals = {}
for secondsStamp in range(epochS, epochE + 1, secondsGranularity):
intervals.update({secondsStamp: dict()})
timestampPattern = 't%Y%m%dT%H%M%S' # TODO: Timezone
for a in data['features']:
name = a['properties']['name']
geometries.update({name: a['geometry']['coordinates']})
for timestamp, value in a['properties'].iteritems():
if timestamp != 'name':
observationStartTime = timestamp[:-4]
secondsTimestamp = int(
time.mktime(
time.strptime(observationStartTime,
timestampPattern)))
for interval in intervals.keys():
if secondsTimestamp >= interval \
and secondsTimestamp < (
interval + secondsGranularity):
if name in intervals[interval].keys():
intervals[interval][name].append(float(value))
else:
intervals[interval].update(
{name: [float(value)]})
break
for interval in intervals.keys():
if len(intervals[interval]) != 0:
timestamp = datetime.datetime.fromtimestamp(interval).strftime(
't%Y%m%dT%H%M%S')
tableName = '{}_{}_{}_{}'.format(options['output'], offering,
key, timestamp)
if ':' in tableName:
tableName = '_'.join(tableName.split(':'))
if '-' in tableName:
tableName = '_'.join(tableName.split('-'))
if '.' in tableName:
tableName = '_'.join(tableName.split('.'))
new = VectorTopo(tableName)
if overwrite() is True:
try:
new.remove()
except:
pass
new.open(mode='w',
layer=1,
tab_name=tableName,
link_name=tableName,
tab_cols=cols,
overwrite=True)
i = 0
for procedure, values in intervals[interval].iteritems():
if new.exist() is False:
i = 1
else:
i += 1
if options['method'] == 'average':
value = sum(values) / len(values)
elif options['method'] == 'sum':
value = sum(values)
# TODO: Other aggregations methods
new.write(Point(*geometries[procedure]),
cat=i,
attrs=(
procedure,
value,
))
new.table.conn.commit()
new.close(build=False)
run_command('v.build', quiet=True, map=tableName)
if options['bbox'] == '':
run_command('g.region', vect=tableName, res=resolution)
run_command('v.to.rast',
input=tableName,
output=tableName,
use='attr',
attribute_column='value',
layer=1,
label_column='name',
type='point',
quiet=True)
if flags['k'] is False:
run_command('g.remove',
'f',
type='vector',
name=tableName,
quiet=True)
def main():
punti=options['points']
dem=options['dem']
TR=options['time']
outlet=options['outlets']
outlets = outlet.split(',')
try:
TR=int(TR)
except:
print 'TR is not a number'
sys.exit()
grass.use_temp_region()
#test to access points
new_points = VectorTopo(punti)
if new_points.exist():
points_str=grass.read_command('v.to.db', map=punti,type='point',option='coor',flags='p')
points_list = points_str.split('\n')
points_list.remove('')
else:
print 'File %s does not exist' % punti
sys.exit()
#get region in order to estimate the threshold as 1/1000 of total cells
grass.run_command('g.region',raster=dem)
regione=grass.region()
thrshold=float(regione['cells'])/300
#stream and drainage determination
grass.run_command('r.watershed', elevation=dem, threshold=500, stream='raster_streams', drainage='drainage',overwrite=True,flags='s')
#the radius is little more than the current resolution
radius=regione['nsres']*1.4
output_points = []
'''
STARTIN CICLE ON EACH OUTLET IN LIST
'''
category=1
for outlet in points_list[1:]:
outlet = outlet.split('|')[1:-1]
print ', '.join(outlet)
grass.run_command('g.region',raster=dem)
grass.run_command('r.circle', output='circle', coordinate=','.join(outlet), max=radius,overwrite=True)
#get the distances and take the shortest distance
distances=grass.read_command('r.distance', map='circle,raster_streams')
list_dist=distances.split('\n')
list_dist.remove('')
list_tuple=[]
for distance in list_dist:
dist=distance.split(':')
my_tupla=dist[0],dist[1],float(dist[2]),dist[3],dist[4],dist[5],dist[6]
list_tuple.append(my_tupla)
tuple_orderedByDistance=sorted(list_tuple, key=lambda distanza: distanza[2])
del(distances,list_tuple,list_dist)
#calculate the basin and read its statistics
outlet=tuple_orderedByDistance[0][-2:]
xoutlet=float(outlet[0])
youtlet=float(outlet[1])
grass.run_command('r.water.outlet',input='drainage',output='basin',coordinates=str(xoutlet)+','+str(youtlet) , overwrite=True)
statistics=grass.read_command('r.univar',map=dem, zones='basin')
main_stat=statistics.splitlines()[-9:]
#order the stream network
grass.run_command('r.mask',raster='basin')
grass.run_command('r.stream.order',stream_rast='raster_streams', direction='drainage', elevation=dem,horton='horton',overwrite=True)
stream_stat=grass.read_command('r.stream.stats', stream_rast='horton', direction='drainage', elevation=dem,flags='o')
network_statistics=stream_stat.split('\n')
network_statistics.remove('')
#get the max order
network_statistics[-1].split()
total_length=float(network_statistics[-1].split(',')[2])
area_basin=float(network_statistics[-1].split(',')[3])
area_basin_Ha=area_basin*100
mean_elev=float(main_stat[3].split(':')[-1])
min_elev=float(main_stat[0].split(':')[-1])
max_elev=float(main_stat[1].split(':')[-1])
deltaH=max_elev-min_elev
average_slope=float(network_statistics[-1].split(',')[4])
grass.run_command('r.mask',flags='r')
TcGiandotti=(4*np.sqrt(area_basin)+1.5*total_length)/(0.8*np.sqrt(mean_elev-min_elev))
TcKirpich=0.945*(total_length**3./deltaH)**0.385
if area_basin_Ha > 1000: #TODO controlla i riferimenti
corrivazione = TcGiandotti
grass.info('using giandotti')
grass.info(str(TcGiandotti))
formula = 'Giandotti'
else:
corrivazione = TcKirpich
formula = 'Kirpich'
grass.info('using Kirpich')
grass.info(str(TcKirpich))
if corrivazione < 24:
aPar='a24@PERMANENT'
bPar='b24@PERMANENT'
kPar='k24@PERMANENT'
else:
aPar='a15@PERMANENT'
bPar='b15@PERMANENT'
kPar='k15@PERMANENT'
CNmap = 'CN@PERMANENT'
corrivazione=TcGiandotti
aStat=grass.read_command('r.univar',map=aPar, zones='basin')
aMain_stat=aStat.splitlines()[12].split(':')[-1]
aMain_stat=float(aMain_stat)
bStat=grass.read_command('r.univar',map=bPar, zones='basin')
bMain_stat=bStat.splitlines()[12].split(':')[-1]
bMain_stat=float(bMain_stat)
kStat=grass.read_command('r.univar',map=kPar, zones='basin')
kMain_stat=kStat.splitlines()[12].split(':')[-1]
kMain_stat=float(kMain_stat)
CNstat = grass.read_command('r.univar',map=CNmap, zones='basin')
CN=CNstat.splitlines()[12].split(':')[-1]
CN=float(CN)
g.message('area basin in km2: ')
print area_basin
print 'mean elev: '
print mean_elev-min_elev
print 'delta H:'
print deltaH
print 'total reach length: '
print total_length
print 'a mean:'
print aMain_stat
print '\n b mean: '
print bMain_stat
print '\n k mean: '
print kMain_stat
print 'CN mean:'
CN = 70.12/82.63 * CN
print CN
f_K_T = 1-kMain_stat*(0.45+0.799*np.log(-np.log(1-1./TR)))
print 'f(k,T): '
print f_K_T
h=f_K_T*aMain_stat*corrivazione**bMain_stat
print '\n h main:'
print h
X1 = 100*corrivazione/(0.236+0.062*corrivazione)
X2 = 0.003*corrivazione+0.0234
Pa = 100 - area_basin_Ha/(X1+X2*area_basin_Ha)
Ha = h*Pa/100
new = VectorTopo('outlet')
S1 = (1000./CN)-10
Pn = (Ha-5.08*S1)**2/(Ha+20.32*S1)
Qc = (1/360.)*Pn*area_basin_Ha/corrivazione
print 'discharge: '
print Qc
#print table.columns.types()
#[u'INTEGER', u'TEXT', u'integer', u'double precision']
'''
------------------------------
START CALCULATION OF LOCAL UPSTREAM SLOPE
------------------------------
'''
#offsets for moving windows
offsets = [d
for j in xrange(1,1+1)
for i in [j,-j]
for d in [(i,0),(0,i),(i,i),(i,-i)]]
#rename dtm as elevation for future calculation if not exist
if not VectorTopo('elevation').exist():
grass.run_command('g.rename',raster="%s,elevation" % dem)
elev_renamed=True
#define drainage direction
drainage_incoming = [2,4,3,1,6,8,7,5]
drainage_outcoming = []
diag_dist= (regione['nsres']**2+regione['ewres']**2)**0.5
# [(1, 0), (0, 1), (1, 1), (1, -1), (-1, 0), (0, -1), (-1, -1), (-1, 1),
cell_dists = [regione['nsres'],
regione['ewres'],
diag_dist,
diag_dist,
regione['nsres'],
regione['ewres'],
diag_dist,
diag_dist
]
# define the calculation term
terms = ["(drainage[%d,%d] == %d && not(isnull(raster_streams[0,0])) && not(isnull(raster_streams[%d,%d])) )"
% ((offsets[j]+tuple([drainage_incoming[j]])+offsets[j]))
for j in range(len(drainage_incoming))]
#define the operation expression
terms_calc = [ "(elevation[%d,%d] - elevation) * %s"
% (offsets[j]+(terms[j],) ) for j in range(len(terms))]
terms_calc_slope = [ "( (elevation[%d,%d] - elevation)/%10.4f ) * %s"
% (offsets[j]+(cell_dists[j],)+(terms[j],)) for j in range(len(terms))]
expr = "num_cells_drain_into = (%s)" % " + ".join(terms)
expr1 = "elevation_percentile4 = if(isnull(raster_streams),null(),(%s))" % " + ".join(terms)
expr2 = "elevdiff_drain_into = %s" % " + ".join(terms_calc)
expr3 = "slope_drain_into = %s" % " + ".join(terms_calc_slope)
# do the r.mapcalc calculation with the moving window
# exclude the num_cell_calculation_into
#grass.mapcalc( expr )
#print expr2
#grass.mapcalc( expr2 , overwrite=True)
#print expr3
grass.mapcalc( expr3 , overwrite=True)
'''
------------------------------
START CALCULATION OF 2KM UPSTREAM SLOPE
------------------------------
'''
#create an outlet vector
new_outlet = VectorTopo('outlet')
COLS = [(u'cat', 'INTEGER PRIMARY KEY')]
new_outlet.open('w', tab_name='outlet', tab_cols=COLS)
new_outlet.write(Point( xoutlet , youtlet ), cat=1, )
new_outlet.table.conn.commit()
new_outlet.table.execute().fetchall()
new_outlet.close()
#find local main channel
horton_order=grass.raster_what('horton', [[ xoutlet , youtlet ]])
horton_order = int( horton_order[0]['horton']['value'] )
print "Horton order for main channel:"
print horton_order
grass.run_command('g.region', zoom='horton')
grass.mapcalc( "main_stream = if((horton == %d),1,null())" % horton_order, overwrite=True )
grass.run_command('r.to.vect', input='main_stream', output='main_stream', type='line',overwrite=True)
grass.run_command('v.build.polylines', overwrite=True, input='main_stream', output='main_stream_poly', cats='first')
#network analysis on main channel
grass.run_command('v.net',input='main_stream_poly', points='outlet', output='main_stream_connected', operation='connect', threshold=radius*3,overwrite=True)
grass.run_command('v.net.iso', input='main_stream_connected',output='main_stream_iso', center_cats=1, costs='100,200,400',overwrite=True)
report=grass.read_command('v.category', input='main_stream_iso', option='report',type='line')
min_max = report.split('\n')[3].split()[-2:]
min_cat = int (min_max[0] )
max_cat = int (min_max[1] )
elev_outlet = grass.raster_what('elevation', [[ xoutlet , youtlet ]])
elev_outlet = float( elev_outlet[0]['elevation']['value'] )
drops = []
for i in range(min_cat,max_cat):
grass.run_command('v.extract',input='main_stream_iso' ,type='line',
cats=i, output='main_stream_%s' % i,overwrite=True)
grass.run_command('v.to.points', input='main_stream_%s' % i,type='line',
output='nodes',use='node',overwrite=True)
points=grass.read_command('v.to.db',flags='p', map='nodes', type='point',
option='coor', columns='x,y', layer=2)
points=points.split('\n')[1:]
points.remove('')
elevations_drops = []
print points
for point in points:
xpoint = float ( point.split('|')[1] )
ypoint = float( point.split('|')[2] )
elev = grass.raster_what('elevation', [[ xpoint , ypoint ]])
elev = float( elev[0]['elevation']['value'] )
elevations_drops.append(elev-elev_outlet)
elevations_drops.sort(reverse=True)
drops.append(elevations_drops[0])
print 'list di drops:'
print drops
#sample the raster slope in the outlets
slope_query=grass.raster_what('slope_drain_into', [[ xoutlet , youtlet ]])
slope = slope_query[0]['slope_drain_into']['value']
if slope == '0':
slope = 1./10000
else:
slope = float( slope )
dict_output = {
'xoutlet':xoutlet,
'youtlet':youtlet,
'cat':category,
'attrs':(Qc, slope, 9810.0*Qc*slope,total_length,elev_outlet,drops[0],drops[1],drops[2],drops[0]/100.,drops[1]/200.,drops[2]/400.,)
}
output_points.append(dict_output)
print category
category+=1
print category
#cleaning part
if elev_renamed:
grass.run_command('g.rename',raster='elevation,%s' % dem)
grass.del_temp_region()
grass.run_command('g.remove',flags='f', type='raster', name='main_stream,basin,circle,drainage,horton,raster_streams,slope_drain_into')
grass.run_command('g.remove',flags='f', type='vector', name='main_stream,nodes,outlet')
grass.run_command('g.remove',type='vector',pattern='main_stream*',flags='f')
#creation of output data container
print output_points
new = VectorTopo('output')
COLS = [(u'cat', 'INTEGER PRIMARY KEY'), (u'discharge', u'double precision') ,
(u'local_upslope', u'double precision'), (u'TSP_local', u'double precision'),
(u'ch_len', u'double precision'),(u'elev', u'double precision'),
(u'drop100', u'double precision'),
(u'drop200', u'double precision'), (u'drop400', u'double precision'),
(u'upslope_100', u'double precision'),
(u'upslope_200', u'double precision'),(u'upslope_400', u'double precision')
]
new.open('w', tab_name='output', tab_cols=COLS)
for elem in output_points:
new.write(Point( elem['xoutlet'],
elem['youtlet'] ),
cat = elem['cat'],
attrs=elem['attrs']
)
new.table.conn.commit()
new.table.execute().fetchall()
new.close()
def main():
try:
from pygbif import occurrences
from pygbif import species
except ImportError:
grass.fatal(_("Cannot import pygbif (https://github.com/sckott/pygbif)"
" library."
" Please install it (pip install pygbif)"
" or ensure that it is on path"
" (use PYTHONPATH variable)."))
# Parse input options
output = options['output']
mask = options['mask']
species_maps = flags['i']
no_region_limit = flags['r']
no_topo = flags['b']
print_species = flags['p']
print_species_table = flags['t']
print_species_shell = flags['g']
print_occ_number = flags['o']
allow_no_geom = flags['n']
hasGeoIssue = flags['s']
taxa_list = options['taxa'].split(',')
institutionCode = options['institutioncode']
basisofrecord = options['basisofrecord']
recordedby = options['recordedby'].split(',')
date_from = options['date_from']
date_to = options['date_to']
country = options['country']
continent = options['continent']
rank = options['rank']
# Define static variable
#Initialize cat
cat = 0
# Number of occurrences to fetch in one request
chunk_size = 300
# lat/lon proj string
latlon_crs = ['+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0.000,0.000,0.000',
'+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0,0,0,0,0,0,0']
# List attributes available in Darwin Core
# not all attributes are returned in each request
# to avoid key errors when accessing the dictionary returned by pygbif
# presence of DWC keys in the returned dictionary is checked using this list
# The number of keys in this list has to be equal to the number of columns
# in the attribute table and the attributes written for each occurrence
dwc_keys = ['key', 'taxonRank', 'taxonKey', 'taxonID', 'scientificName',
'species', 'speciesKey', 'genericName', 'genus', 'genusKey',
'family', 'familyKey', 'order', 'orderKey', 'class',
'classKey', 'phylum', 'phylumKey', 'kingdom', 'kingdomKey',
'eventDate', 'verbatimEventDate', 'startDayOfYear',
'endDayOfYear', 'year', 'month', 'day', 'occurrenceID',
'occurrenceStatus', 'occurrenceRemarks', 'Habitat',
'basisOfRecord', 'preparations', 'sex', 'type', 'locality',
'verbatimLocality', 'decimalLongitude', 'decimalLatitude',
'geodeticDatum', 'higerGeography', 'continent', 'country',
'countryCode', 'stateProvince', 'gbifID', 'protocol',
'identifier', 'recordedBy', 'identificationID', 'identifiers',
'dateIdentified', 'modified', 'institutionCode',
'lastInterpreted', 'lastParsed', 'references', 'relations',
'catalogNumber', 'occurrenceDetails', 'datasetKey',
'datasetName', 'collectionCode', 'rights', 'rightsHolder',
'license', 'publishingOrgKey', 'publishingCountry',
'lastCrawled', 'specificEpithet', 'facts', 'issues',
'extensions', 'language']
# Deinfe columns for attribute table
cols = [('cat', 'INTEGER PRIMARY KEY'),
('g_search', 'varchar(100)'),
('g_key', 'integer'),
('g_taxonrank', 'varchar(50)'),
('g_taxonkey', 'integer'),
('g_taxonid', 'varchar(50)'),
('g_scientificname', 'varchar(255)'),
('g_species', 'varchar(255)'),
('g_specieskey', 'integer'),
('g_genericname', 'varchar(255)'),
('g_genus', 'varchar(50)'),
('g_genuskey', 'integer'),
('g_family', 'varchar(50)'),
('g_familykey', 'integer'),
('g_order', 'varchar(50)'),
('g_orderkey', 'integer'),
('g_class', 'varchar(50)'),
('g_classkey', 'integer'),
('g_phylum', 'varchar(50)'),
('g_phylumkey', 'integer'),
('g_kingdom', 'varchar(50)'),
('g_kingdomkey', 'integer'),
('g_eventdate', 'text'),
('g_verbatimeventdate', 'varchar(50)'),
('g_startDayOfYear', 'integer'),
('g_endDayOfYear', 'integer'),
('g_year', 'integer'),
('g_month', 'integer'),
('g_day', 'integer'),
('g_occurrenceid', 'varchar(255)'),
('g_occurrenceStatus', 'varchar(50)'),
('g_occurrenceRemarks', 'varchar(50)'),
('g_Habitat', 'varchar(50)'),
('g_basisofrecord', 'varchar(50)'),
('g_preparations', 'varchar(50)'),
('g_sex', 'varchar(50)'),
('g_type', 'varchar(50)'),
('g_locality', 'varchar(255)'),
('g_verbatimlocality', 'varchar(255)'),
('g_decimallongitude', 'double precision'),
('g_decimallatitude', 'double precision'),
('g_geodeticdatum', 'varchar(50)'),
('g_higerGeography', 'varchar(255)'),
('g_continent', 'varchar(50)'),
('g_country', 'varchar(50)'),
('g_countryCode', 'varchar(50)'),
('g_stateProvince', 'varchar(50)'),
('g_gbifid', 'varchar(255)'),
('g_protocol', 'varchar(255)'),
('g_identifier', 'varchar(50)'),
('g_recordedby', 'varchar(255)'),
('g_identificationid', 'varchar(255)'),
('g_identifiers', 'text'),
('g_dateidentified', 'text'),
('g_modified', 'text'),
('g_institutioncode', 'varchar(50)'),
('g_lastinterpreted', 'text'),
('g_lastparsed', 'text'),
('g_references', 'varchar(255)'),
('g_relations', 'text'),
('g_catalognumber', 'varchar(50)'),
('g_occurrencedetails', 'text'),
('g_datasetkey', 'varchar(50)'),
('g_datasetname', 'varchar(255)'),
('g_collectioncode', 'varchar(50)'),
('g_rights', 'varchar(255)'),
('g_rightsholder', 'varchar(255)'),
('g_license', 'varchar(50)'),
('g_publishingorgkey', 'varchar(50)'),
('g_publishingcountry', 'varchar(50)'),
('g_lastcrawled', 'text'),
('g_specificepithet', 'varchar(50)'),
('g_facts', 'text'),
('g_issues', 'text'),
('g_extensions', 'text'),
('g_language', 'varchar(50)')]
set_output_encoding()
# Set temporal filter if requested by user
# Initialize eventDate filter
eventDate = None
# Check if date from is compatible (ISO compliant)
if date_from:
try:
parse(date_from)
except:
grass.fatal("Invalid invalid start date provided")
if date_from and not date_to:
eventDate = '{}'.format(date_from)
# Check if date to is compatible (ISO compliant)
if date_to:
try:
parse(date_to)
except:
grass.fatal("Invalid invalid end date provided")
# Check if date to is after date_from
if parse(date_from) < parse(date_to):
eventDate = '{},{}'.format(date_from, date_to)
else:
grass.fatal("Invalid date range: End date has to be after start date!")
# Set filter on basisOfRecord if requested by user
if basisofrecord == 'ALL':
basisOfRecord = None
else:
basisOfRecord = basisofrecord
# Allow also occurrences with spatial issues if requested by user
hasGeospatialIssue = False
if hasGeoIssue:
hasGeospatialIssue = True
# Allow also occurrences without coordinates if requested by user
hasCoordinate = True
if allow_no_geom:
hasCoordinate = False
# Set reprojection parameters
# Set target projection of current LOCATION
target_crs = grass.read_command('g.proj', flags='fj').rstrip(os.linesep)
target = osr.SpatialReference(target_crs)
target.ImportFromProj4(target_crs)
if target == 'XY location (unprojected)':
grass.fatal("Sorry, XY locations are not supported!")
# Set source projection from GBIF
source = osr.SpatialReference()
source.ImportFromEPSG(4326)
if target_crs not in latlon_crs:
transform = osr.CoordinateTransformation(source, target)
reverse_transform = osr.CoordinateTransformation(target, source)
# Generate WKT polygon to use for spatial filtering if requested
if mask:
if len(mask.split('@')) == 2:
m = VectorTopo(mask.split('@')[0], mapset=mask.split('@')[1])
else:
m = VectorTopo(mask)
if not m.exist():
grass.fatal('Could not find vector map <{}>'.format(mask))
m.open('r')
if not m.is_open():
grass.fatal('Could not open vector map <{}>'.format(mask))
# Use map Bbox as spatial filter if map contains <> 1 area
if m.number_of('areas') == 1:
region_pol = [area.to_wkt() for area in m.viter("areas")][0]
else:
bbox = str(m.bbox()).replace('Bbox(', '').replace(' ', '').rstrip(')').split(',')
region_pol = 'POLYGON(({0} {1}, {0} {3}, {2} {3}, {2} {1}, {0} {1}))'.format(bbox[2],
bbox[0], bbox[3], bbox[1])
m.close()
else:
# Do not limit import spatially if LOCATION is able to take global data
if no_region_limit:
if target_crs not in latlon_crs:
grass.fatal('Import of data from outside the current region is'
'only supported in a WGS84 location!')
region_pol = None
else:
# Limit import spatially to current region
# if LOCATION is !NOT! able to take global data
# to avoid pprojection ERRORS
region = grass.parse_command('g.region', flags='g')
region_pol = 'POLYGON(({0} {1}, {0} {3}, {2} {3}, {2} {1}, {0} {1}))'.format(region['e'],
region['n'], region['w'], region['s'])
# Do not reproject in latlon LOCATIONS
if target_crs not in latlon_crs:
pol = ogr.CreateGeometryFromWkt(region_pol)
pol.Transform(reverse_transform)
pol = pol.ExportToWkt()
else:
pol = region_pol
# Create output map if not output maps for each species are requested
if not species_maps and not print_species and not print_species_shell and not print_occ_number and not print_species_table:
mapname = output
new = Vector(mapname)
new.open('w', tab_name=mapname, tab_cols=cols)
cat = 1
# Import data for each species
for s in taxa_list:
# Get the taxon key if not the taxon key is provided as input
try:
key = int(s)
except:
try:
species_match = species.name_backbone(s, rank=rank,
strict=False,
verbose=True)
key = species_match['usageKey']
except:
grass.error('Data request for taxon {} failed. Are you online?'.format(s))
continue
# Return matching taxon and alternatives and exit
if print_species:
print('Matching taxon for {} is:'.format(s))
print('{} {}'.format(species_match['scientificName'], species_match['status']))
if 'alternatives' in list(species_match.keys()):
print('Alternative matches might be:'.format(s))
for m in species_match['alternatives']:
print('{} {}'.format(m['scientificName'], m['status']))
else:
print('No alternatives found for the given taxon')
continue
if print_species_shell:
print('match={}'.format(species_match['scientificName']))
if 'alternatives' in list(species_match.keys()):
alternatives = []
for m in species_match['alternatives']:
alternatives.append(m['scientificName'])
print('alternatives={}'.format(','.join(alternatives)))
continue
if print_species_table:
if 'alternatives' in list(species_match.keys()):
if len(species_match['alternatives']) == 0:
print('{0}|{1}|{2}|'.format(s, key, species_match['scientificName']))
else:
alternatives = []
for m in species_match['alternatives']:
alternatives.append(m['scientificName'])
print('{0}|{1}|{2}|{3}'.format(s, key, species_match['scientificName'],
','.join(alternatives)))
continue
try:
returns_n = occurrences.search(taxonKey=key,
hasGeospatialIssue=hasGeospatialIssue,
hasCoordinate=hasCoordinate,
institutionCode=institutionCode,
basisOfRecord=basisOfRecord,
recordedBy=recordedby,
eventDate=eventDate,
continent=continent,
country=country,
geometry=pol,
limit=1)['count']
except:
grass.error('Data request for taxon {} faild. Are you online?'.format(s))
returns_n = 0
# Exit if search does not give a return
# Print only number of returns for the given search and exit
if print_occ_number:
grass.message('Found {0} occurrences for taxon {1}...'.format(returns_n, s))
continue
elif returns_n <= 0:
grass.warning('No occurrences for current search for taxon {0}...'.format(s))
continue
elif returns_n >= 200000:
grass.warning('Your search for {1} returns {0} records.\n'
'Unfortunately, the GBIF search API is limited to 200,000 records per request.\n'
'The download will be incomplete. Please consider to split up your search.'.format(returns_n, s))
# Get the number of chunks to download
chunks = int(math.ceil(returns_n / float(chunk_size)))
grass.verbose('Downloading {0} occurrences for taxon {1}...'.format(returns_n, s))
# Create a map for each species if requested using map name as suffix
if species_maps:
mapname = '{}_{}'.format(s.replace(' ', '_'), output)
new = Vector(mapname)
new.open('w', tab_name=mapname, tab_cols=cols)
cat = 0
# Download the data from GBIF
for c in range(chunks):
# Define offset
offset = c * chunk_size
# Adjust chunk_size to the hard limit of 200,000 records in GBIF API
# if necessary
if offset + chunk_size >= 200000:
chunk_size = 200000 - offset
# Get the returns for the next chunk
returns = occurrences.search(taxonKey=key,
hasGeospatialIssue=hasGeospatialIssue,
hasCoordinate=hasCoordinate,
institutionCode=institutionCode,
basisOfRecord=basisOfRecord,
recordedBy=recordedby,
eventDate=eventDate,
continent=continent,
country=country,
geometry=pol,
limit=chunk_size,
offset=offset)
# Write the returned data to map and attribute table
for res in returns['results']:
if target_crs not in latlon_crs:
point = ogr.CreateGeometryFromWkt('POINT ({} {})'.format(res['decimalLongitude'], res['decimalLatitude']))
point.Transform(transform)
x = point.GetX()
y = point.GetY()
else:
x = res['decimalLongitude']
y = res['decimalLatitude']
point = Point(x, y)
for k in dwc_keys:
if k not in list(res.keys()):
res.update({k: None})
cat = cat + 1
new.write(point, cat=cat, attrs=(
'{}'.format(s),
res['key'],
res['taxonRank'],
res['taxonKey'],
res['taxonID'],
res['scientificName'],
res['species'],
res['speciesKey'],
res['genericName'],
res['genus'],
res['genusKey'],
res['family'],
res['familyKey'],
res['order'],
res['orderKey'],
res['class'],
res['classKey'],
res['phylum'],
res['phylumKey'],
res['kingdom'],
res['kingdomKey'],
'{}'.format(res['eventDate']) if res['eventDate'] else None,
'{}'.format(res['verbatimEventDate']) if res['verbatimEventDate'] else None,
res['startDayOfYear'],
res['endDayOfYear'],
res['year'],
res['month'],
res['day'],
res['occurrenceID'],
res['occurrenceStatus'],
res['occurrenceRemarks'],
res['Habitat'],
res['basisOfRecord'],
res['preparations'],
res['sex'],
res['type'],
res['locality'],
res['verbatimLocality'],
res['decimalLongitude'],
res['decimalLatitude'],
res['geodeticDatum'],
res['higerGeography'],
res['continent'],
res['country'],
res['countryCode'],
res['stateProvince'],
res['gbifID'],
res['protocol'],
res['identifier'],
res['recordedBy'],
res['identificationID'],
','.join(res['identifiers']),
'{}'.format(res['dateIdentified']) if res['dateIdentified'] else None,
'{}'.format(res['modified']) if res['modified'] else None,
res['institutionCode'],
'{}'.format(res['lastInterpreted']) if res['lastInterpreted'] else None,
'{}'.format(res['lastParsed']) if res['lastParsed'] else None,
res['references'],
','.join(res['relations']),
res['catalogNumber'],
'{}'.format(res['occurrenceDetails']) if res['occurrenceDetails'] else None,
res['datasetKey'],
res['datasetName'],
res['collectionCode'],
res['rights'],
res['rightsHolder'],
res['license'],
res['publishingOrgKey'],
res['publishingCountry'],
'{}'.format(res['lastCrawled']) if res['lastCrawled'] else None,
res['specificEpithet'],
','.join(res['facts']),
','.join(res['issues']),
','.join(res['extensions']),
res['language'],))
cat = cat + 1
# Close the current map if a map for each species is requested
if species_maps:
new.table.conn.commit()
new.close()
if not no_topo:
grass.run_command('v.build', map=mapname, option='build')
# Close the output map if not a map for each species is requested
if not species_maps and not print_species and not print_species_shell and not print_occ_number and not print_species_table:
new.table.conn.commit()
new.close()
if not no_topo:
grass.run_command('v.build', map=mapname, option='build')
def full_maps(parsed_obs, offering, seconds_granularity, resolution,
event_time, target):
"""Create raster maps.
Maps represent represent offerings, observed props and procedures
:param parsed_obs: Observations for a given offering in geoJSON format
:param offering: A collection of sensors used to conveniently group them up
:param seconds_granularity: Granularity in seconds
:param resolution: 2D grid resolution for rasterization
:param event_time: Timestamp of first/of last requested observation
:param target:
"""
timestamp_pattern = '%Y-%m-%dT%H:%M:%S' # TODO: Timezone
start_time = event_time.split('+')[0]
epoch_s = int(time.mktime(time.strptime(start_time, timestamp_pattern)))
end_time = event_time.split('+')[1].split('/')[1]
epoch_e = int(time.mktime(time.strptime(end_time, timestamp_pattern)))
for key, observation in parsed_obs.items():
print('Creating raster maps for offering '
'{}, observed property {}'.format(offering, key))
data = json.loads(observation)
crs = data['crs']
crs = int(crs['properties']['name'].split(':')[-1])
transform = soslib.get_transformation(crs, target)
cols = [(u'cat', 'INTEGER PRIMARY KEY'), (u'name', 'VARCHAR'),
(u'value', 'DOUBLE')]
geometries = dict()
intervals = {}
for secondsStamp in range(epoch_s, epoch_e + 1, seconds_granularity):
intervals.update({secondsStamp: dict()})
timestamp_pattern = 't%Y%m%dT%H%M%S' # TODO: Timezone
for a in data['features']:
name = a['properties']['name']
sx, sy, sz = a['geometry']['coordinates']
point = ogr.CreateGeometryFromWkt('POINT ({} {} {})'.format(
sx, sy, sz))
point.Transform(transform)
coords = (point.GetX(), point.GetY(), point.GetZ())
geometries.update({name: coords})
for timestamp, value in a['properties'].items():
if timestamp != 'name':
observation_start_time = timestamp[:-4]
seconds_timestamp = int(
time.mktime(
time.strptime(observation_start_time,
timestamp_pattern)))
for interval in intervals.keys():
if interval <= seconds_timestamp < (
interval + seconds_granularity):
if name in intervals[interval].keys():
intervals[interval][name].append(float(value))
else:
intervals[interval].update(
{name: [float(value)]})
break
for interval in intervals.keys():
if len(intervals[interval]) != 0:
timestamp = datetime.datetime.fromtimestamp(interval).strftime(
't%Y%m%dT%H%M%S')
table_name = '{}_{}_{}_{}'.format(options['output'], offering,
key, timestamp)
if ':' in table_name:
table_name = '_'.join(table_name.split(':'))
if '-' in table_name:
table_name = '_'.join(table_name.split('-'))
if '.' in table_name:
table_name = '_'.join(table_name.split('.'))
new = VectorTopo(table_name)
if overwrite() is True:
try:
new.remove()
except:
pass
new.open(mode='w',
layer=1,
tab_name=table_name,
link_name=table_name,
tab_cols=cols,
overwrite=True)
i = 0
n = None
s = None
e = None
w = None
for procedure, values in intervals[interval].items():
if new.exist() is False:
i = 1
else:
i += 1
if options['method'] == 'average':
value = sum(values) / len(values)
elif options['method'] == 'sum':
value = sum(values)
# TODO: Other aggregations methods
new.write(Point(*geometries[procedure]),
cat=i,
attrs=(
procedure,
value,
))
if options['bbox'] == '':
x, y, z = geometries[procedure]
if not n:
n = y + resolution / 2
s = y - resolution / 2
e = x + resolution / 2
w = x - resolution / 2
else:
if y >= n:
n = y + resolution / 2
if y <= s:
s = y - resolution / 2
if x >= e:
e = x + resolution / 2
if x <= w:
w = x - resolution / 2
new.table.conn.commit()
new.close(build=False)
run_command('v.build', quiet=True, map=table_name)
if options['bbox'] == '':
run_command('g.region', n=n, s=s, w=w, e=e, res=resolution)
run_command('v.to.rast',
input=table_name,
output=table_name,
use='attr',
attribute_column='value',
layer=1,
type='point',
quiet=True)
if flags['k'] is False:
run_command('g.remove',
'f',
type='vector',
name=table_name,
quiet=True)
def test_all(self):
self.assertModule(
"v.stream.order",
input="stream_network",
points="stream_network_outlets",
output="stream_network_order_test_all",
threshold=25,
order=["strahler", "shreve", "drwal", "scheidegger"],
overwrite=True,
verbose=True,
)
# Check all values
v = VectorTopo(name="stream_network_order_test_all", mapset="")
v.open(mode="r")
self.assertTrue(v.exist(), True)
self.assertEqual(v.num_primitive_of("line"), 101)
# feature 4
self.assertEqual(v.read(4).attrs.cat, 41)
self.assertEqual(v.read(4).attrs["outlet_cat"], 1)
self.assertEqual(v.read(4).attrs["network"], 1)
self.assertEqual(v.read(4).attrs["reversed"], 0)
self.assertEqual(v.read(4).attrs["strahler"], 4)
self.assertEqual(v.read(4).attrs["shreve"], 32)
self.assertEqual(v.read(4).attrs["drwal"], 6)
self.assertEqual(v.read(4).attrs["scheidegger"], 64)
v.close()
# Check for column copy
self.assertModule(
"v.stream.order",
input="stream_network_order_test_all",
points="stream_network_outlets",
output="stream_network_order_test_all_2",
threshold=25,
order=["strahler", "shreve", "drwal", "scheidegger"],
columns=["strahler", "shreve", "drwal", "scheidegger"],
overwrite=True,
verbose=True,
)
# Check all values and their copies
v = VectorTopo(name="stream_network_order_test_all_2", mapset="")
v.open(mode="r")
self.assertTrue(v.exist(), True)
self.assertEqual(v.num_primitive_of("line"), 101)
# feature 4
self.assertEqual(v.read(4).attrs.cat, 4)
self.assertEqual(v.read(4).attrs["outlet_cat"], 1)
self.assertEqual(v.read(4).attrs["network"], 1)
self.assertEqual(v.read(4).attrs["reversed"], 0)
self.assertEqual(v.read(4).attrs["strahler"], 4)
self.assertEqual(v.read(4).attrs["shreve"], 32)
self.assertEqual(v.read(4).attrs["drwal"], 6)
self.assertEqual(v.read(4).attrs["scheidegger"], 64)
self.assertEqual(v.read(4).attrs["strahler_1"], 4)
self.assertEqual(v.read(4).attrs["shreve_1"], 32)
self.assertEqual(v.read(4).attrs["drwal_1"], 6)
self.assertEqual(v.read(4).attrs["scheidegger_1"], 64)
# feature 7
self.assertEqual(v.read(7).attrs.cat, 7)
self.assertEqual(v.read(7).attrs["outlet_cat"], 1)
self.assertEqual(v.read(7).attrs["network"], 1)
self.assertEqual(v.read(7).attrs["reversed"], 0)
self.assertEqual(v.read(7).attrs["strahler"], 2)
self.assertEqual(v.read(7).attrs["strahler_1"], 2)
self.assertEqual(v.read(7).attrs["shreve"], 4)
self.assertEqual(v.read(7).attrs["drwal"], 3)
self.assertEqual(v.read(7).attrs["scheidegger"], 8)
v.close()
