def export_results(vect_name,
results,
cats,
rlayer,
training=None,
cols=None,
overwrite=False,
append=False,
pkl=None):
if pkl:
res = open(pkl, 'w')
pickle.dump(results, res)
res.close()
# check if the link already exist
with Vector(vect_name, mode='r') as vct:
link = vct.dblinks.by_name(rlayer)
mode = 'r' if link else 'w'
print("Opening vector <%s>" % vect_name)
with Vector(vect_name, mode=mode) as vect:
if cols:
cols.insert(0, COLS[0])
tab = link.table() if link else Table(rlayer, vect.table.conn)
if tab.exist() and append:
columns_to_up = []
# add the column to the table
for cname, ctype in cols:
columns_to_up.append("%s=?" % cname)
if cname not in tab.columns:
tab.columns.add(cname, ctype)
upsql = "UPDATE %s SET %s WHERE %s=%s"
up = upsql % (tab.name, ','.join(columns_to_up), tab.key, '?')
else:
if tab.exist():
print("Table <%s> already exist, will be removed." %
tab.name)
tab.drop(force=True)
print("Ceating a new table <%s>." % rlayer)
tab.create(cols)
up = ''
export2onesqlite(tab, cats.astype(int), up,
*[cls['predict'].astype(int) for cls in results])
if mode == 'w':
nlyr = len(vect.dblinks) + 1
link = Link(nlyr, tab.name, tab.name)
vect.dblinks.add(link)
vect.build()
else:
for cls in results:
create_tab(vect, cls['name'], cats, cls['predict'], training,
COLS if training else COLS[:2])
def main():
mapset = Mapset()
mapset.current()
with open(options['output'], 'w') as fd:
for rast in mapset.glist('raster', pattern='*_B04_10m'):
items = rast.split('_')
d = datetime.strptime(items[2], '%Y%m%dT%H%M%S')
## workaround
dd = d + timedelta(seconds=1)
vect = '{}_{}_MSK_CLOUDS'.format(items[1], items[2])
mask_vect = '{}_{}'.format(vect, options['map'].split('@')[0])
if Vector(vect).exist():
Module('v.overlay',
ainput=options['map'],
binput=vect,
operator='not',
output=mask_vect)
else:
copy(options['map'], mask_vect, 'vector')
Module('r.mask', vector=mask_vect, overwrite=True)
Module('g.remove', flags='f', type='vector', name=mask_vect)
Module('g.rename', raster=['MASK', mask_vect])
fd.write("{0}|{1}|{2}{3}".format(mask_vect,
d.strftime('%Y-%m-%d %H:%M:%S'),
dd.strftime('%Y-%m-%d %H:%M:%S'),
os.linesep))
return 0
def extract_training(vect, tvect, tlayer):
"""Assign a class to all the areas that contained, are contained
or intersect a training vector"""
msgr = get_msgr()
tname, tmset = tvect.split('@') if '@' in tvect else (tvect, '')
vname, vmset = vect.split('@') if '@' in vect else (vect, '')
with VectorTopo(tname, tmset, mode='r') as trn:
with VectorTopo(vname, vmset, mode='r') as vct:
layer_num, layer_name = get_layer_num_name(vct, tlayer)
# instantiate the area objects
trn_area = Area(c_mapinfo=trn.c_mapinfo)
seg_area = Area(c_mapinfo=vct.c_mapinfo)
n_areas = trn.number_of('areas')
# check/remove/create a new table
table, create_link = make_new_table(vct, layer_name, force=True)
find_lines(table, [l for l in trn.viter('lines')], vct)
# find and save all the segments
find_area(table, trn.viter('areas', idonly=True), trn_area,
seg_area, n_areas, vct)
check_balance(table, trn.table)
if create_link:
msgr.message(_("Connect the new table to the vector map..."))
with Vector(vect, mode='rw') as seg:
link = Link(layer_num, name=layer_name, table=table.name)
seg.dblinks.add(link)
seg.build()
def main(opt, flg):
#
# Set check variables
#
overwrite = True
rasters = opt['rasters'].split(',') if opt['rasters'] else []
rprefix = opt['rprefix'].split(',') if opt['rprefix'] else []
def split(x): return x.split('@') if '@' in x else (x, '')
vname, vmset = split(opt['vector'])
shpcsv = opt['shpcsv'] if opt['shpcsv'] else vname + '.csv'
rstcsv = (opt['rstcsv'].split(',') if opt['rstcsv']
else [split(rst)[0] + '.csv' for rst in rasters])
zones = opt['zones'] if opt['zones'] else vname + '_zones'
nprocs = int(opt.get('nprocs', 1))
if rasters:
if rprefix and len(rasters) != len(rprefix):
raise
if len(rasters) != len(rstcsv):
raise
prefixes = rprefix if rprefix else rasters
skipshp = opt['skipshape'].split(',') if opt['skipshape'] else []
skiprst = opt['skipunivar'].split(',') if opt['skipunivar'] else []
layer = int(opt['layer'])
newlayer = int(opt['newlayer'])
newlayername = (opt['newlayername'] if opt['newlayername']
else vname + '_stats')
newtabname = opt['newtabname'] if opt['newtabname'] else vname + '_stats'
rstpercentile = float(opt['rstpercentile'])
separator = opt.get('separator', ';')
#
# compute
#
if not os.path.exists(shpcsv):
get_shp_csv(opt['vector'], shpcsv, overwrite, separator)
if not get_mapset_raster(zones):
get_zones(opt['vector'], zones, layer)
if not rstcsv or not os.path.exists(rstcsv[0]):
get_rst_csv(rasters, zones, rstcsv, rstpercentile, overwrite,
nprocs, separator)
newlink = Link(newlayer, newlayername, newtabname)
newtab = newlink.table()
with Vector(vname, vmset, mode='r', layer=layer) as vct:
mode = 'r' if newlink in vct.dblinks else 'rw'
with VectorTopo(vname, vmset, mode=mode, layer=layer) as vct:
update_cols(newtab, shpcsv, rstcsv, prefixes, skipshp, skiprst, separator=separator)
if mode == 'rw':
# add the new link
vct.dblinks.add(newlink)
vct.build()
def _get_streamlist(self, stream):
"""Compute shape of stream.
:param stream:
"""
Module(
'db.copy',
from_table=self.tab_stream_shape,
from_database='$GISDBASE/$LOCATION_NAME/PERMANENT/sqlite/sqlite.db',
to_table='stream_shape')
# TODO: (check if smoderp column exists)
sfields = ["number", "shapetype", "b", "m", "roughness", "q365"]
try:
self._join_table(stream, self.tab_stream_shape_code,
'stream_shape', self.tab_stream_shape_code,
sfields)
except:
self._add_field(stream, "smoderp", "TEXT", "0")
self._join_table(stream, self.tab_stream_shape_code,
'stream_shape', self.tab_stream_shape_code,
sfields)
sfields.insert(1, "smoderp")
# TODO: rewrite into pygrass syntax
ret = Module('v.db.select',
flags='c',
map=stream,
columns=sfields,
stdout_=PIPE)
for row in ret.outputs.stdout.splitlines():
for value in row.split('|'):
if value == '':
raise StreamPreparationError(
"Empty value in {} found.".format(stream))
with Vector(stream) as data:
self.field_names = data.table.columns.names()
self.stream_tmp = [[] for field in self.field_names]
# TODO: rewrite into pygrass syntax
ret = Module('v.db.select',
flags='c',
map=stream,
columns=self.field_names,
stdout_=PIPE)
for row in ret.outputs.stdout.splitlines():
i = 0
for val in row.split('|'):
self.stream_tmp[i].append(float(val))
i += 1
self.streamlist = []
self._streamlist()
def _add_field(self, vector, newfield, datatype, default_value):
"""
Adds field into attribute field of feature class.
:param vector: Feature class to which new field is to be added.
:param newfield:
:param datatype:
:param default_value:
"""
with Vector(vector) as data:
found = newfield in data.table.columns.names()
if found:
Module('v.db.dropcolumn', map=vector, columns=newfield)
Module('v.db.addcolumn',
map=vector,
columns='{} {}'.format(newfield, datatype))
Module('v.db.update', map=vector, column=newfield, value=default_value)
def extract_classes(vect, layer):
vect, mset = vect.split("@") if "@" in vect else (vect, "")
with Vector(vect, mapset=mset, layer=layer, mode="r") as vct:
vct.table.filters.select("cat", "class")
return {key: val for key, val in vct.table.execute()}
def main():
# Get the options
input = options["input"]
output = options["output"]
strds = options["strds"]
tempwhere = options["t_where"]
where = options["where"]
methods = options["method"]
percentile = options["percentile"]
overwrite = grass.overwrite()
quiet = True
if grass.verbosity() > 2:
quiet = False
if where == "" or where == " " or where == "\n":
where = None
# Check the number of sample strds and the number of columns
strds_names = strds.split(",")
# Make sure the temporal database exists
tgis.init()
# We need a database interface
dbif = tgis.SQLDatabaseInterfaceConnection()
dbif.connect()
samples = []
first_strds = tgis.open_old_stds(strds_names[0], "strds", dbif)
# Single space time raster dataset
if len(strds_names) == 1:
granu = first_strds.get_granularity()
rows = first_strds.get_registered_maps(
"name,mapset,start_time,end_time", tempwhere, "start_time", dbif)
if not rows:
dbif.close()
grass.fatal(
_("Space time raster dataset <%s> is empty") %
first_strds.get_id())
for row in rows:
start = row["start_time"]
end = row["end_time"]
raster_maps = [
row["name"] + "@" + row["mapset"],
]
s = Sample(start, end, raster_maps, first_strds.get_name(), granu)
samples.append(s)
else:
# Multiple space time raster datasets
for name in strds_names[1:]:
dataset = tgis.open_old_stds(name, "strds", dbif)
if dataset.get_temporal_type() != first_strds.get_temporal_type():
grass.fatal(
_(
"Temporal type of space time raster "
"datasets must be equal\n<%(a)s> of type "
"%(type_a)s do not match <%(b)s> of type "
"%(type_b)s" % {
"a": first_strds.get_id(),
"type_a": first_strds.get_temporal_type(),
"b": dataset.get_id(),
"type_b": dataset.get_temporal_type()
}))
mapmatrizes = tgis.sample_stds_by_stds_topology(
"strds", "strds", strds_names, strds_names[0], False, None,
"equal", False, False)
#TODO check granularity for multiple STRDS
for i in range(len(mapmatrizes[0])):
isvalid = True
mapname_list = []
for mapmatrix in mapmatrizes:
entry = mapmatrix[i]
if entry["samples"]:
sample = entry["samples"][0]
name = sample.get_id()
if name is None:
isvalid = False
break
else:
mapname_list.append(name)
if isvalid:
entry = mapmatrizes[0][i]
map = entry["granule"]
start, end = map.get_temporal_extent_as_tuple()
s = Sample(start, end, mapname_list, name)
samples.append(s)
# Get the layer and database connections of the input vector
if where:
try:
grass.run_command("v.extract",
input=input,
where=where,
output=output)
except CalledModuleError:
dbif.close()
grass.fatal(
_("Unable to run v.extract for vector map"
" <%s> and where <%s>") % (input, where))
else:
gcopy(input, output, 'vector')
msgr = Messenger()
perc_curr = 0
perc_tot = len(samples)
pymap = Vector(output)
try:
pymap.open('r')
except:
dbif.close()
grass.fatal(_("Unable to create vector map <%s>" % output))
pymap.close()
for sample in samples:
raster_names = sample.raster_names
# Call v.what.rast for each raster map
for name in raster_names:
day = sample.printDay()
column_name = "%s_%s" % (sample.strds_name, day)
try:
grass.run_command("v.rast.stats",
map=output,
raster=name,
column=column_name,
method=methods,
percentile=percentile,
quiet=quiet,
overwrite=overwrite)
except CalledModuleError:
dbif.close()
grass.fatal(
_("Unable to run v.what.rast for vector map"
" <%s> and raster map <%s>") % (output, name))
msgr.percent(perc_curr, perc_tot, 1)
perc_curr += 1
dbif.close()
def main():
# lazy imports
import grass.temporal as tgis
from grass.pygrass.utils import copy as gcopy
from grass.pygrass.messages import Messenger
from grass.pygrass.vector import Vector
# Get the options
input = options["input"]
output = options["output"]
strds = options["strds"]
where = options["where"]
tempwhere = options["t_where"]
if output and flags['u']:
grass.fatal(_("Cannot combine 'output' option and 'u' flag"))
elif not output and not flags['u']:
grass.fatal(_("'output' option or 'u' flag must be given"))
elif not output and flags['u']:
grass.warning(
_("Attribute table of vector {name} will be updated...").format(
name=input))
if where == "" or where == " " or where == "\n":
where = None
overwrite = grass.overwrite()
quiet = True
if grass.verbosity() > 2:
quiet = False
# Check the number of sample strds and the number of columns
strds_names = strds.split(",")
# Make sure the temporal database exists
tgis.init()
# We need a database interface
dbif = tgis.SQLDatabaseInterfaceConnection()
dbif.connect()
samples = []
first_strds = tgis.open_old_stds(strds_names[0], "strds", dbif)
# Single space time raster dataset
if len(strds_names) == 1:
granu = first_strds.get_granularity()
rows = first_strds.get_registered_maps(
"name,mapset,start_time,end_time", tempwhere, "start_time", dbif)
if not rows:
dbif.close()
grass.fatal(
_("Space time raster dataset <%s> is empty") %
first_strds.get_id())
for row in rows:
start = row["start_time"]
end = row["end_time"]
raster_maps = [
row["name"] + "@" + row["mapset"],
]
s = Sample(start, end, raster_maps, first_strds.get_name(), granu)
samples.append(s)
else:
# Multiple space time raster datasets
for name in strds_names[1:]:
dataset = tgis.open_old_stds(name, "strds", dbif)
if dataset.get_temporal_type() != first_strds.get_temporal_type():
grass.fatal(
_(
"Temporal type of space time raster "
"datasets must be equal\n<%(a)s> of type "
"%(type_a)s do not match <%(b)s> of type "
"%(type_b)s" % {
"a": first_strds.get_id(),
"type_a": first_strds.get_temporal_type(),
"b": dataset.get_id(),
"type_b": dataset.get_temporal_type()
}))
mapmatrizes = tgis.sample_stds_by_stds_topology(
"strds", "strds", strds_names, strds_names[0], False, None,
"equal", False, False)
#TODO check granularity for multiple STRDS
for i in range(len(mapmatrizes[0])):
isvalid = True
mapname_list = []
for mapmatrix in mapmatrizes:
entry = mapmatrix[i]
if entry["samples"]:
sample = entry["samples"][0]
name = sample.get_id()
if name is None:
isvalid = False
break
else:
mapname_list.append(name)
if isvalid:
entry = mapmatrizes[0][i]
map = entry["granule"]
start, end = map.get_temporal_extent_as_tuple()
s = Sample(start, end, mapname_list, name)
samples.append(s)
# Get the layer and database connections of the input vector
if output:
gcopy(input, output, 'vector')
else:
output = input
msgr = Messenger()
perc_curr = 0
perc_tot = len(samples)
pymap = Vector(output)
try:
pymap.open('r')
except:
dbif.close()
grass.fatal(_("Unable to create vector map <%s>" % output))
if len(pymap.dblinks) == 0:
try:
pymap.close()
grass.run_command("v.db.addtable", map=output)
except CalledModuleError:
dbif.close()
grass.fatal(
_("Unable to add table <%s> to vector map <%s>" % output))
if pymap.is_open():
pymap.close()
for sample in samples:
raster_names = sample.raster_names
# Call v.what.rast for each raster map
for name in raster_names:
coltype = "DOUBLE PRECISION"
# Get raster map type
raster_map = tgis.RasterDataset(name)
raster_map.load()
if raster_map.metadata.get_datatype() == "CELL":
coltype = "INT"
day = sample.printDay()
column_name = "%s_%s" % (sample.strds_name, day)
column_string = "%s %s" % (column_name, coltype)
column_string.replace('.', '_')
try:
grass.run_command("v.db.addcolumn",
map=output,
column=column_string,
overwrite=overwrite)
except CalledModuleError:
dbif.close()
grass.fatal(
_("Unable to add column %s to vector map "
"<%s> ") % (column_string, output))
try:
grass.run_command("v.what.rast",
map=output,
raster=name,
column=column_name,
where=where,
quiet=quiet)
except CalledModuleError:
dbif.close()
grass.fatal(
_("Unable to run v.what.rast for vector map"
" <%s> and raster map <%s>") %
(output, str(raster_names)))
msgr.percent(perc_curr, perc_tot, 1)
perc_curr += 1
dbif.close()
def main(opt, flg):
#
# Set check variables
#
overwrite = True
rasters = opt["rasters"].split(",") if opt["rasters"] else []
rprefix = opt["rprefix"].split(",") if opt["rprefix"] else []
def split(x):
return x.split("@") if "@" in x else (x, "")
vname, vmset = split(opt["vector"])
shpcsv = opt["shpcsv"] if opt["shpcsv"] else vname + ".csv"
rstcsv = (opt["rstcsv"].split(",") if opt["rstcsv"] else
[split(rst)[0] + ".csv" for rst in rasters])
zones = opt["zones"] if opt["zones"] else vname + "_zones"
nprocs = int(opt.get("nprocs", 1))
if rasters:
if rprefix and len(rasters) != len(rprefix):
raise
if len(rasters) != len(rstcsv):
raise
prefixes = rprefix if rprefix else rasters
else:
prefixes = None
skipshp = opt["skipshape"].split(",") if opt["skipshape"] else []
skiprst = opt["skipunivar"].split(",") if opt["skipunivar"] else []
layer = int(opt["layer"])
newlayer = int(opt["newlayer"])
newlayername = opt["newlayername"] if opt[
"newlayername"] else vname + "_stats"
newtabname = opt["newtabname"] if opt["newtabname"] else vname + "_stats"
rstpercentile = float(opt["rstpercentile"])
separator = opt.get("separator", ";")
#
# compute
#
if not os.path.exists(shpcsv):
get_shp_csv(opt["vector"], shpcsv, overwrite, separator)
if not get_mapset_raster(zones):
get_zones(opt["vector"], zones, layer)
if not rstcsv or not os.path.exists(rstcsv[0]):
get_rst_csv(rasters, zones, rstcsv, rstpercentile, overwrite, nprocs,
separator)
newlink = Link(newlayer, newlayername, newtabname)
newtab = newlink.table()
with Vector(vname, vmset, mode="r", layer=layer) as vct:
mode = "r" if newlink in vct.dblinks else "rw"
with VectorTopo(vname, vmset, mode=mode, layer=layer) as vct:
update_cols(newtab,
shpcsv,
rstcsv,
prefixes,
skipshp,
skiprst,
separator=separator)
if mode == "rw":
# add the new link
vct.dblinks.add(newlink)
vct.build()
def extract_classes(vect, layer):
vect, mset = vect.split('@') if '@'in vect else (vect, '')
with Vector(vect, mapset=mset, layer=layer, mode='r') as vct:
vct.table.filters.select('cat', 'class')
return {key: val for key, val in vct.table.execute()}
#!/usr/bin/env python3
from grass.pygrass.vector import Vector
obce = Vector('obce_bod', mapset='ruian')
obce.open('r')
for prvek in obce:
print ("{:<25}: {:.0f} {:.0f}".format(prvek.attrs['nazev'], prvek.x, prvek.y))
obce.close()
def main():
try:
import pysptools.eea as eea
except ImportError:
gs.fatal(_("Cannot import pysptools \
(https://pypi.python.org/pypi/pysptools) library."
" Please install it (pip install pysptools)"
" or ensure that it is on path"
" (use PYTHONPATH variable)."))
try:
# sklearn is a dependency of used pysptools functionality
import sklearn
except ImportError:
gs.fatal(_("Cannot import sklearn \
(https://pypi.python.org/pypi/scikit-learn) library."
" Please install it (pip install scikit-learn)"
" or ensure that it is on path"
" (use PYTHONPATH variable)."))
try:
from cvxopt import solvers, matrix
except ImportError:
gs.fatal(_("Cannot import cvxopt \
(https://pypi.python.org/pypi/cvxopt) library."
" Please install it (pip install cvxopt)"
" or ensure that it is on path"
" (use PYTHONPATH variable)."))
# Parse input options
input = options['input']
output = options['output']
prefix = options['prefix']
endmember_n = int(options['endmember_n'])
endmembers = options['endmembers']
if options['maxit']:
maxit = options['maxit']
else:
maxit = 0
extraction_method = options['extraction_method']
unmixing_method = options['unmixing_method']
atgp_init = True if not flags['n'] else False
# List maps in imagery group
try:
maps = gs.read_command('i.group', flags='g', group=input,
quiet=True).rstrip('\n').split('\n')
except:
pass
# Validate input
# q and maxit can be None according to manual, but does not work in current pysptools version
if endmember_n <= 0:
gs.fatal('Number of endmembers has to be > 0')
"""if (extraction_method == 'PPI' or
extraction_method == 'NFINDR'):
gs.fatal('Extraction methods PPI and NFINDR require endmember_n >= 2')
endmember_n = None"""
if maxit <= 0:
maxit = 3 * len(maps)
if endmember_n > len(maps) + 1:
gs.warning('More endmembers ({}) requested than bands in \
input imagery group ({})'.format(endmember_n, len(maps)))
if extraction_method != 'PPI':
gs.fatal('Only PPI method can extract more endmembers than number \
of bands in the imagery group')
if not atgp_init and extraction_method != 'NFINDR':
gs.verbose('ATGP is only taken into account in \
NFINDR extraction method...')
# Get metainformation from input bands
band_types = {}
img = None
n = 0
gs.verbose('Reading imagery group...')
for m in maps:
map = m.split('@')
# Build numpy stack from imagery group
raster = r.raster2numpy(map[0], mapset=map[1])
if raster == np.float64:
raster = float32(raster)
gs.warning('{} is of type Float64.\
Float64 is currently not supported.\
Reducing precision to Float32'.format(raster))
# Determine map type
band_types[map[0]] = get_rastertype(raster)
# Create cube and mask from GRASS internal NoData value
if n == 0:
img = raster
# Create mask from GRASS internal NoData value
mask = mask_rasternd(raster)
else:
img = np.dstack((img, raster))
mask = np.logical_and((mask_rasternd(raster)), mask)
n = n + 1
# Read a mask if present and give waringing if not
# Note that otherwise NoData is read as values
gs.verbose('Checking for MASK...')
try:
MASK = r.raster2numpy('MASK', mapset=getenv('MAPSET')) == 1
mask = np.logical_and(MASK, mask)
MASK = None
except:
pass
if extraction_method == 'NFINDR':
# Extract endmembers from valid pixels using NFINDR function from pysptools
gs.verbose('Extracting endmembers using NFINDR...')
nfindr = eea.NFINDR()
E = nfindr.extract(img, endmember_n, maxit=maxit, normalize=False,
ATGP_init=atgp_init, mask=mask)
elif extraction_method == 'PPI':
# Extract endmembers from valid pixels using PPI function from pysptools
gs.verbose('Extracting endmembers using PPI...')
ppi = eea.PPI()
E = ppi.extract(img, endmember_n, numSkewers=10000, normalize=False,
mask=mask)
elif extraction_method == 'FIPPI':
# Extract endmembers from valid pixels using FIPPI function from pysptools
gs.verbose('Extracting endmembers using FIPPI...')
fippi = eea.FIPPI()
# q and maxit can be None according to manual, but does not work
"""if not maxit and not endmember_n:
E = fippi.extract(img, q=None, normalize=False, mask=mask)
if not maxit:
E = fippi.extract(img, q=endmember_n, normalize=False, mask=mask)
if not endmember_n:
E = fippi.extract(img, q=int(), maxit=maxit, normalize=False,
mask=mask)
else:
E = fippi.extract(img, q=endmember_n, maxit=maxit, normalize=False,
mask=mask)"""
E = fippi.extract(img, q=endmember_n, maxit=maxit, normalize=False,
mask=mask)
# Write output file in format required for i.spec.unmix addon
if output:
gs.verbose('Writing spectra file...')
n = 0
with open(output, 'w') as o:
o.write('# Channels: {}\n'.format('\t'.join(band_types.keys())))
o.write('# Wrote {} spectra line wise.\n#\n'.format(endmember_n))
o.write('Matrix: {0} by {1}\n'.format(endmember_n, len(maps)))
for e in E:
o.write('row{0}: {1}\n'.format(n, '\t'.join([str(i) for i in e])))
n = n + 1
# Write vector map with endmember information if requested
if endmembers:
gs.verbose('Writing vector map with endmembers...')
from grass.pygrass import utils as u
from grass.pygrass.gis.region import Region
from grass.pygrass.vector import Vector
from grass.pygrass.vector import VectorTopo
from grass.pygrass.vector.geometry import Point
# Build attribute table
# Deinfe columns for attribute table
cols = [(u'cat', 'INTEGER PRIMARY KEY')]
for b in band_types.keys():
cols.append((b.replace('.','_'), band_types[b]))
# Get region information
reg = Region()
# Create vector map
new = Vector(endmembers)
new.open('w', tab_name=endmembers, tab_cols=cols)
cat = 1
for e in E:
# Get indices
idx = np.where((img[:,:]==e).all(-1))
# Numpy array is ordered rows, columns (y,x)
if len(idx[0]) == 0 or len(idx[1]) == 0:
gs.warning('Could not compute coordinated for endmember {}. \
Please consider rescaling your data to integer'.format(cat))
cat = cat + 1
continue
coords = u.pixel2coor((idx[1][0], idx[0][0]), reg)
point = Point(coords[1] + reg.ewres / 2.0,
coords[0] - reg.nsres / 2.0)
# Get attributes
n = 0
attr = []
for b in band_types.keys():
if band_types[b] == u'INTEGER':
attr.append(int(e[n]))
else:
attr.append(float(e[n]))
n = n + 1
# Write geometry with attributes
new.write(point, cat=cat,
attrs=tuple(attr))
cat = cat + 1
# Close vector map
new.table.conn.commit()
new.close(build=True)
if prefix:
# Run spectral unmixing
import pysptools.abundance_maps as amaps
if unmixing_method == 'FCLS':
fcls = amaps.FCLS()
result = fcls.map(img, E, normalize=False, mask=mask)
elif unmixing_method == 'NNLS':
nnls = amaps.NNLS()
result = nnls.map(img, E, normalize=False, mask=mask)
elif unmixing_method == 'UCLS':
ucls = amaps.UCLS()
result = ucls.map(img, E, normalize=False, mask=mask)
# Write results
for l in range(endmember_n):
rastname = '{0}_{1}'.format(prefix, l + 1)
r.numpy2raster(result[:,:,l], 'FCELL', rastname)
def main():
# Get the options
input = options["input"]
output = options["output"]
strds = options["strds"]
where = options["where"]
tempwhere = options["t_where"]
if output and flags['u']:
grass.fatal(_("Cannot combine 'output' option and 'u' flag"))
elif not output and not flags['u']:
grass.fatal(_("'output' option or 'u' flag must be given"))
elif not output and flags['u']:
grass.warning(_("Attribute table of vector {name} will be updated...").format(name=input))
if where == "" or where == " " or where == "\n":
where = None
overwrite = grass.overwrite()
quiet = True
if grass.verbosity() > 2:
quiet = False
# Check the number of sample strds and the number of columns
strds_names = strds.split(",")
# Make sure the temporal database exists
tgis.init()
# We need a database interface
dbif = tgis.SQLDatabaseInterfaceConnection()
dbif.connect()
samples = []
first_strds = tgis.open_old_stds(strds_names[0], "strds", dbif)
# Single space time raster dataset
if len(strds_names) == 1:
rows = first_strds.get_registered_maps("name,mapset,start_time,end_time",
tempwhere, "start_time",
dbif)
if not rows:
dbif.close()
grass.fatal(_("Space time raster dataset <%s> is empty") %
first_strds.get_id())
for row in rows:
start = row["start_time"]
end = row["end_time"]
raster_maps = [row["name"] + "@" + row["mapset"], ]
s = Sample(start, end, raster_maps, first_strds.get_name())
samples.append(s)
else:
# Multiple space time raster datasets
for name in strds_names[1:]:
dataset = tgis.open_old_stds(name, "strds", dbif)
if dataset.get_temporal_type() != first_strds.get_temporal_type():
grass.fatal(_("Temporal type of space time raster "
"datasets must be equal\n<%(a)s> of type "
"%(type_a)s do not match <%(b)s> of type "
"%(type_b)s" % {"a": first_strds.get_id(),
"type_a": first_strds.get_temporal_type(),
"b": dataset.get_id(),
"type_b": dataset.get_temporal_type()}))
mapmatrizes = tgis.sample_stds_by_stds_topology("strds", "strds",
strds_names,
strds_names[0],
False, None,
"equal", False,
False)
for i in xrange(len(mapmatrizes[0])):
isvalid = True
mapname_list = []
for mapmatrix in mapmatrizes:
entry = mapmatrix[i]
if entry["samples"]:
sample = entry["samples"][0]
name = sample.get_id()
if name is None:
isvalid = False
break
else:
mapname_list.append(name)
if isvalid:
entry = mapmatrizes[0][i]
map = entry["granule"]
start, end = map.get_temporal_extent_as_tuple()
s = Sample(start, end, mapname_list, name)
samples.append(s)
# Get the layer and database connections of the input vector
if output:
gcopy(input, output, 'vector')
else:
output = input
msgr = Messenger()
perc_curr = 0
perc_tot = len(samples)
pymap = Vector(output)
try:
pymap.open('r')
except:
dbif.close()
grass.fatal(_("It is not possible to open the new map %s" % output))
if len(pymap.dblinks) == 0:
try:
grass.run_command("v.db.addtable", map=output)
except CalledModuleError:
dbif.close()
grass.fatal(_("Impossible add table to vector %s" % output))
pymap.close()
for sample in samples:
raster_names = sample.raster_names
# Call v.what.rast for each raster map
for name in raster_names:
coltype = "DOUBLE PRECISION"
# Get raster map type
raster_map = tgis.RasterDataset(name)
raster_map.load()
if raster_map.metadata.get_datatype() == "CELL":
coltype = "INT"
day = sample.printDay()
column_name = "%s_%s" % (sample.strds_name, day)
column_string = "%s %s" % (column_name, coltype)
column_string.replace('.', '_')
try:
grass.run_command("v.db.addcolumn", map=output,
column=column_string,
overwrite=overwrite)
except CalledModuleError:
dbif.close()
grass.fatal(_("Unable to add column %s to vector map "
"<%s> ") % (column_string, output))
try:
grass.run_command("v.what.rast", map=output, raster=name,
column=column_name, where=where,
quiet=quiet)
except CalledModuleError:
dbif.close()
grass.fatal(_("Unable to run v.what.rast for vector map"
" <%s> and raster map <%s>") %
(output, str(raster_names)))
msgr.percent(perc_curr, perc_tot, 1)
perc_curr += 1
dbif.close()
def get_colors(vtraining):
vect, mset = vtraining.split("@") if "@" in vtraining else (vtraining, "")
with Vector(vect, mapset=mset, mode="r") as vct:
cur = vct.table.execute("SELECT cat, color FROM %s;" % vct.name)
return dict([c for c in cur.fetchall()])
def main():
try:
from pygbif import occurrences
from pygbif import species
except ImportError:
grass.fatal(_("Cannot import pygbif (https://github.com/sckott/pygbif)"
" library."
" Please install it (pip install pygbif)"
" or ensure that it is on path"
" (use PYTHONPATH variable)."))
# Parse input options
output = options['output']
mask = options['mask']
species_maps = flags['i']
no_region_limit = flags['r']
no_topo = flags['b']
print_species = flags['p']
print_species_table = flags['t']
print_species_shell = flags['g']
print_occ_number = flags['o']
allow_no_geom = flags['n']
hasGeoIssue = flags['s']
taxa_list = options['taxa'].split(',')
institutionCode = options['institutioncode']
basisofrecord = options['basisofrecord']
recordedby = options['recordedby'].split(',')
date_from = options['date_from']
date_to = options['date_to']
country = options['country']
continent = options['continent']
rank = options['rank']
# Define static variable
#Initialize cat
cat = 0
# Number of occurrences to fetch in one request
chunk_size = 300
# lat/lon proj string
latlon_crs = ['+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0.000,0.000,0.000',
'+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0,0,0,0,0,0,0']
# List attributes available in Darwin Core
# not all attributes are returned in each request
# to avoid key errors when accessing the dictionary returned by pygbif
# presence of DWC keys in the returned dictionary is checked using this list
# The number of keys in this list has to be equal to the number of columns
# in the attribute table and the attributes written for each occurrence
dwc_keys = ['key', 'taxonRank', 'taxonKey', 'taxonID', 'scientificName',
'species', 'speciesKey', 'genericName', 'genus', 'genusKey',
'family', 'familyKey', 'order', 'orderKey', 'class',
'classKey', 'phylum', 'phylumKey', 'kingdom', 'kingdomKey',
'eventDate', 'verbatimEventDate', 'startDayOfYear',
'endDayOfYear', 'year', 'month', 'day', 'occurrenceID',
'occurrenceStatus', 'occurrenceRemarks', 'Habitat',
'basisOfRecord', 'preparations', 'sex', 'type', 'locality',
'verbatimLocality', 'decimalLongitude', 'decimalLatitude',
'geodeticDatum', 'higerGeography', 'continent', 'country',
'countryCode', 'stateProvince', 'gbifID', 'protocol',
'identifier', 'recordedBy', 'identificationID', 'identifiers',
'dateIdentified', 'modified', 'institutionCode',
'lastInterpreted', 'lastParsed', 'references', 'relations',
'catalogNumber', 'occurrenceDetails', 'datasetKey',
'datasetName', 'collectionCode', 'rights', 'rightsHolder',
'license', 'publishingOrgKey', 'publishingCountry',
'lastCrawled', 'specificEpithet', 'facts', 'issues',
'extensions', 'language']
# Deinfe columns for attribute table
cols = [('cat', 'INTEGER PRIMARY KEY'),
('g_search', 'varchar(100)'),
('g_key', 'integer'),
('g_taxonrank', 'varchar(50)'),
('g_taxonkey', 'integer'),
('g_taxonid', 'varchar(50)'),
('g_scientificname', 'varchar(255)'),
('g_species', 'varchar(255)'),
('g_specieskey', 'integer'),
('g_genericname', 'varchar(255)'),
('g_genus', 'varchar(50)'),
('g_genuskey', 'integer'),
('g_family', 'varchar(50)'),
('g_familykey', 'integer'),
('g_order', 'varchar(50)'),
('g_orderkey', 'integer'),
('g_class', 'varchar(50)'),
('g_classkey', 'integer'),
('g_phylum', 'varchar(50)'),
('g_phylumkey', 'integer'),
('g_kingdom', 'varchar(50)'),
('g_kingdomkey', 'integer'),
('g_eventdate', 'text'),
('g_verbatimeventdate', 'varchar(50)'),
('g_startDayOfYear', 'integer'),
('g_endDayOfYear', 'integer'),
('g_year', 'integer'),
('g_month', 'integer'),
('g_day', 'integer'),
('g_occurrenceid', 'varchar(255)'),
('g_occurrenceStatus', 'varchar(50)'),
('g_occurrenceRemarks', 'varchar(50)'),
('g_Habitat', 'varchar(50)'),
('g_basisofrecord', 'varchar(50)'),
('g_preparations', 'varchar(50)'),
('g_sex', 'varchar(50)'),
('g_type', 'varchar(50)'),
('g_locality', 'varchar(255)'),
('g_verbatimlocality', 'varchar(255)'),
('g_decimallongitude', 'double precision'),
('g_decimallatitude', 'double precision'),
('g_geodeticdatum', 'varchar(50)'),
('g_higerGeography', 'varchar(255)'),
('g_continent', 'varchar(50)'),
('g_country', 'varchar(50)'),
('g_countryCode', 'varchar(50)'),
('g_stateProvince', 'varchar(50)'),
('g_gbifid', 'varchar(255)'),
('g_protocol', 'varchar(255)'),
('g_identifier', 'varchar(50)'),
('g_recordedby', 'varchar(255)'),
('g_identificationid', 'varchar(255)'),
('g_identifiers', 'text'),
('g_dateidentified', 'text'),
('g_modified', 'text'),
('g_institutioncode', 'varchar(50)'),
('g_lastinterpreted', 'text'),
('g_lastparsed', 'text'),
('g_references', 'varchar(255)'),
('g_relations', 'text'),
('g_catalognumber', 'varchar(50)'),
('g_occurrencedetails', 'text'),
('g_datasetkey', 'varchar(50)'),
('g_datasetname', 'varchar(255)'),
('g_collectioncode', 'varchar(50)'),
('g_rights', 'varchar(255)'),
('g_rightsholder', 'varchar(255)'),
('g_license', 'varchar(50)'),
('g_publishingorgkey', 'varchar(50)'),
('g_publishingcountry', 'varchar(50)'),
('g_lastcrawled', 'text'),
('g_specificepithet', 'varchar(50)'),
('g_facts', 'text'),
('g_issues', 'text'),
('g_extensions', 'text'),
('g_language', 'varchar(50)')]
set_output_encoding()
# Set temporal filter if requested by user
# Initialize eventDate filter
eventDate = None
# Check if date from is compatible (ISO compliant)
if date_from:
try:
parse(date_from)
except:
grass.fatal("Invalid invalid start date provided")
if date_from and not date_to:
eventDate = '{}'.format(date_from)
# Check if date to is compatible (ISO compliant)
if date_to:
try:
parse(date_to)
except:
grass.fatal("Invalid invalid end date provided")
# Check if date to is after date_from
if parse(date_from) < parse(date_to):
eventDate = '{},{}'.format(date_from, date_to)
else:
grass.fatal("Invalid date range: End date has to be after start date!")
# Set filter on basisOfRecord if requested by user
if basisofrecord == 'ALL':
basisOfRecord = None
else:
basisOfRecord = basisofrecord
# Allow also occurrences with spatial issues if requested by user
hasGeospatialIssue = False
if hasGeoIssue:
hasGeospatialIssue = True
# Allow also occurrences without coordinates if requested by user
hasCoordinate = True
if allow_no_geom:
hasCoordinate = False
# Set reprojection parameters
# Set target projection of current LOCATION
target_crs = grass.read_command('g.proj', flags='fj').rstrip(os.linesep)
target = osr.SpatialReference(target_crs)
target.ImportFromProj4(target_crs)
if target == 'XY location (unprojected)':
grass.fatal("Sorry, XY locations are not supported!")
# Set source projection from GBIF
source = osr.SpatialReference()
source.ImportFromEPSG(4326)
if target_crs not in latlon_crs:
transform = osr.CoordinateTransformation(source, target)
reverse_transform = osr.CoordinateTransformation(target, source)
# Generate WKT polygon to use for spatial filtering if requested
if mask:
if len(mask.split('@')) == 2:
m = VectorTopo(mask.split('@')[0], mapset=mask.split('@')[1])
else:
m = VectorTopo(mask)
if not m.exist():
grass.fatal('Could not find vector map <{}>'.format(mask))
m.open('r')
if not m.is_open():
grass.fatal('Could not open vector map <{}>'.format(mask))
# Use map Bbox as spatial filter if map contains <> 1 area
if m.number_of('areas') == 1:
region_pol = [area.to_wkt() for area in m.viter("areas")][0]
else:
bbox = str(m.bbox()).replace('Bbox(', '').replace(' ', '').rstrip(')').split(',')
region_pol = 'POLYGON(({0} {1}, {0} {3}, {2} {3}, {2} {1}, {0} {1}))'.format(bbox[2],
bbox[0], bbox[3], bbox[1])
m.close()
else:
# Do not limit import spatially if LOCATION is able to take global data
if no_region_limit:
if target_crs not in latlon_crs:
grass.fatal('Import of data from outside the current region is'
'only supported in a WGS84 location!')
region_pol = None
else:
# Limit import spatially to current region
# if LOCATION is !NOT! able to take global data
# to avoid pprojection ERRORS
region = grass.parse_command('g.region', flags='g')
region_pol = 'POLYGON(({0} {1}, {0} {3}, {2} {3}, {2} {1}, {0} {1}))'.format(region['e'],
region['n'], region['w'], region['s'])
# Do not reproject in latlon LOCATIONS
if target_crs not in latlon_crs:
pol = ogr.CreateGeometryFromWkt(region_pol)
pol.Transform(reverse_transform)
pol = pol.ExportToWkt()
else:
pol = region_pol
# Create output map if not output maps for each species are requested
if not species_maps and not print_species and not print_species_shell and not print_occ_number and not print_species_table:
mapname = output
new = Vector(mapname)
new.open('w', tab_name=mapname, tab_cols=cols)
cat = 1
# Import data for each species
for s in taxa_list:
# Get the taxon key if not the taxon key is provided as input
try:
key = int(s)
except:
try:
species_match = species.name_backbone(s, rank=rank,
strict=False,
verbose=True)
key = species_match['usageKey']
except:
grass.error('Data request for taxon {} failed. Are you online?'.format(s))
continue
# Return matching taxon and alternatives and exit
if print_species:
print('Matching taxon for {} is:'.format(s))
print('{} {}'.format(species_match['scientificName'], species_match['status']))
if 'alternatives' in list(species_match.keys()):
print('Alternative matches might be:'.format(s))
for m in species_match['alternatives']:
print('{} {}'.format(m['scientificName'], m['status']))
else:
print('No alternatives found for the given taxon')
continue
if print_species_shell:
print('match={}'.format(species_match['scientificName']))
if 'alternatives' in list(species_match.keys()):
alternatives = []
for m in species_match['alternatives']:
alternatives.append(m['scientificName'])
print('alternatives={}'.format(','.join(alternatives)))
continue
if print_species_table:
if 'alternatives' in list(species_match.keys()):
if len(species_match['alternatives']) == 0:
print('{0}|{1}|{2}|'.format(s, key, species_match['scientificName']))
else:
alternatives = []
for m in species_match['alternatives']:
alternatives.append(m['scientificName'])
print('{0}|{1}|{2}|{3}'.format(s, key, species_match['scientificName'],
','.join(alternatives)))
continue
try:
returns_n = occurrences.search(taxonKey=key,
hasGeospatialIssue=hasGeospatialIssue,
hasCoordinate=hasCoordinate,
institutionCode=institutionCode,
basisOfRecord=basisOfRecord,
recordedBy=recordedby,
eventDate=eventDate,
continent=continent,
country=country,
geometry=pol,
limit=1)['count']
except:
grass.error('Data request for taxon {} faild. Are you online?'.format(s))
returns_n = 0
# Exit if search does not give a return
# Print only number of returns for the given search and exit
if print_occ_number:
grass.message('Found {0} occurrences for taxon {1}...'.format(returns_n, s))
continue
elif returns_n <= 0:
grass.warning('No occurrences for current search for taxon {0}...'.format(s))
continue
elif returns_n >= 200000:
grass.warning('Your search for {1} returns {0} records.\n'
'Unfortunately, the GBIF search API is limited to 200,000 records per request.\n'
'The download will be incomplete. Please consider to split up your search.'.format(returns_n, s))
# Get the number of chunks to download
chunks = int(math.ceil(returns_n / float(chunk_size)))
grass.verbose('Downloading {0} occurrences for taxon {1}...'.format(returns_n, s))
# Create a map for each species if requested using map name as suffix
if species_maps:
mapname = '{}_{}'.format(s.replace(' ', '_'), output)
new = Vector(mapname)
new.open('w', tab_name=mapname, tab_cols=cols)
cat = 0
# Download the data from GBIF
for c in range(chunks):
# Define offset
offset = c * chunk_size
# Adjust chunk_size to the hard limit of 200,000 records in GBIF API
# if necessary
if offset + chunk_size >= 200000:
chunk_size = 200000 - offset
# Get the returns for the next chunk
returns = occurrences.search(taxonKey=key,
hasGeospatialIssue=hasGeospatialIssue,
hasCoordinate=hasCoordinate,
institutionCode=institutionCode,
basisOfRecord=basisOfRecord,
recordedBy=recordedby,
eventDate=eventDate,
continent=continent,
country=country,
geometry=pol,
limit=chunk_size,
offset=offset)
# Write the returned data to map and attribute table
for res in returns['results']:
if target_crs not in latlon_crs:
point = ogr.CreateGeometryFromWkt('POINT ({} {})'.format(res['decimalLongitude'], res['decimalLatitude']))
point.Transform(transform)
x = point.GetX()
y = point.GetY()
else:
x = res['decimalLongitude']
y = res['decimalLatitude']
point = Point(x, y)
for k in dwc_keys:
if k not in list(res.keys()):
res.update({k: None})
cat = cat + 1
new.write(point, cat=cat, attrs=(
'{}'.format(s),
res['key'],
res['taxonRank'],
res['taxonKey'],
res['taxonID'],
res['scientificName'],
res['species'],
res['speciesKey'],
res['genericName'],
res['genus'],
res['genusKey'],
res['family'],
res['familyKey'],
res['order'],
res['orderKey'],
res['class'],
res['classKey'],
res['phylum'],
res['phylumKey'],
res['kingdom'],
res['kingdomKey'],
'{}'.format(res['eventDate']) if res['eventDate'] else None,
'{}'.format(res['verbatimEventDate']) if res['verbatimEventDate'] else None,
res['startDayOfYear'],
res['endDayOfYear'],
res['year'],
res['month'],
res['day'],
res['occurrenceID'],
res['occurrenceStatus'],
res['occurrenceRemarks'],
res['Habitat'],
res['basisOfRecord'],
res['preparations'],
res['sex'],
res['type'],
res['locality'],
res['verbatimLocality'],
res['decimalLongitude'],
res['decimalLatitude'],
res['geodeticDatum'],
res['higerGeography'],
res['continent'],
res['country'],
res['countryCode'],
res['stateProvince'],
res['gbifID'],
res['protocol'],
res['identifier'],
res['recordedBy'],
res['identificationID'],
','.join(res['identifiers']),
'{}'.format(res['dateIdentified']) if res['dateIdentified'] else None,
'{}'.format(res['modified']) if res['modified'] else None,
res['institutionCode'],
'{}'.format(res['lastInterpreted']) if res['lastInterpreted'] else None,
'{}'.format(res['lastParsed']) if res['lastParsed'] else None,
res['references'],
','.join(res['relations']),
res['catalogNumber'],
'{}'.format(res['occurrenceDetails']) if res['occurrenceDetails'] else None,
res['datasetKey'],
res['datasetName'],
res['collectionCode'],
res['rights'],
res['rightsHolder'],
res['license'],
res['publishingOrgKey'],
res['publishingCountry'],
'{}'.format(res['lastCrawled']) if res['lastCrawled'] else None,
res['specificEpithet'],
','.join(res['facts']),
','.join(res['issues']),
','.join(res['extensions']),
res['language'],))
cat = cat + 1
# Close the current map if a map for each species is requested
if species_maps:
new.table.conn.commit()
new.close()
if not no_topo:
grass.run_command('v.build', map=mapname, option='build')
# Close the output map if not a map for each species is requested
if not species_maps and not print_species and not print_species_shell and not print_occ_number and not print_species_table:
new.table.conn.commit()
new.close()
if not no_topo:
grass.run_command('v.build', map=mapname, option='build')
#!/usr/bin/env python
from grass.pygrass.vector import Vector
obce = Vector('obce_bod', mapset='ruian')
obce.open('r')
for prvek in obce:
print (u"{:<25}: {:.0f} {:.0f}".format(prvek.attrs['nazev'], prvek.x, prvek.y))
obce.close()
def main():
from dateutil.parser import parse
try:
from pygbif import occurrences
from pygbif import species
except ImportError:
grass.fatal(
_("Cannot import pygbif (https://github.com/sckott/pygbif)"
" library."
" Please install it (pip install pygbif)"
" or ensure that it is on path"
" (use PYTHONPATH variable)."))
# Parse input options
output = options["output"]
mask = options["mask"]
species_maps = flags["i"]
no_region_limit = flags["r"]
no_topo = flags["b"]
print_species = flags["p"]
print_species_table = flags["t"]
print_species_shell = flags["g"]
print_occ_number = flags["o"]
allow_no_geom = flags["n"]
hasGeoIssue = flags["s"]
taxa_list = options["taxa"].split(",")
institutionCode = options["institutioncode"]
basisofrecord = options["basisofrecord"]
recordedby = options["recordedby"].split(",")
date_from = options["date_from"]
date_to = options["date_to"]
country = options["country"]
continent = options["continent"]
rank = options["rank"]
# Define static variable
# Initialize cat
cat = 0
# Number of occurrences to fetch in one request
chunk_size = 300
# lat/lon proj string
latlon_crs = [
"+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0.000,0.000,0.000",
"+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0,0,0,0,0,0,0",
"+proj=longlat +no_defs +a=6378137 +rf=298.257223563 +towgs84=0.000,0.000,0.000 +type=crs",
]
# List attributes available in Darwin Core
# not all attributes are returned in each request
# to avoid key errors when accessing the dictionary returned by pygbif
# presence of DWC keys in the returned dictionary is checked using this list
# The number of keys in this list has to be equal to the number of columns
# in the attribute table and the attributes written for each occurrence
dwc_keys = [
"key",
"taxonRank",
"taxonKey",
"taxonID",
"scientificName",
"species",
"speciesKey",
"genericName",
"genus",
"genusKey",
"family",
"familyKey",
"order",
"orderKey",
"class",
"classKey",
"phylum",
"phylumKey",
"kingdom",
"kingdomKey",
"eventDate",
"verbatimEventDate",
"startDayOfYear",
"endDayOfYear",
"year",
"month",
"day",
"occurrenceID",
"occurrenceStatus",
"occurrenceRemarks",
"Habitat",
"basisOfRecord",
"preparations",
"sex",
"type",
"locality",
"verbatimLocality",
"decimalLongitude",
"decimalLatitude",
"coordinateUncertaintyInMeters",
"geodeticDatum",
"higerGeography",
"continent",
"country",
"countryCode",
"stateProvince",
"gbifID",
"protocol",
"identifier",
"recordedBy",
"identificationID",
"identifiers",
"dateIdentified",
"modified",
"institutionCode",
"lastInterpreted",
"lastParsed",
"references",
"relations",
"catalogNumber",
"occurrenceDetails",
"datasetKey",
"datasetName",
"collectionCode",
"rights",
"rightsHolder",
"license",
"publishingOrgKey",
"publishingCountry",
"lastCrawled",
"specificEpithet",
"facts",
"issues",
"extensions",
"language",
]
# Deinfe columns for attribute table
cols = [
("cat", "INTEGER PRIMARY KEY"),
("g_search", "varchar(100)"),
("g_key", "integer"),
("g_taxonrank", "varchar(50)"),
("g_taxonkey", "integer"),
("g_taxonid", "varchar(50)"),
("g_scientificname", "varchar(255)"),
("g_species", "varchar(255)"),
("g_specieskey", "integer"),
("g_genericname", "varchar(255)"),
("g_genus", "varchar(50)"),
("g_genuskey", "integer"),
("g_family", "varchar(50)"),
("g_familykey", "integer"),
("g_order", "varchar(50)"),
("g_orderkey", "integer"),
("g_class", "varchar(50)"),
("g_classkey", "integer"),
("g_phylum", "varchar(50)"),
("g_phylumkey", "integer"),
("g_kingdom", "varchar(50)"),
("g_kingdomkey", "integer"),
("g_eventdate", "text"),
("g_verbatimeventdate", "varchar(50)"),
("g_startDayOfYear", "integer"),
("g_endDayOfYear", "integer"),
("g_year", "integer"),
("g_month", "integer"),
("g_day", "integer"),
("g_occurrenceid", "varchar(255)"),
("g_occurrenceStatus", "varchar(50)"),
("g_occurrenceRemarks", "varchar(50)"),
("g_Habitat", "varchar(50)"),
("g_basisofrecord", "varchar(50)"),
("g_preparations", "varchar(50)"),
("g_sex", "varchar(50)"),
("g_type", "varchar(50)"),
("g_locality", "varchar(255)"),
("g_verbatimlocality", "varchar(255)"),
("g_decimallongitude", "double precision"),
("g_decimallatitude", "double precision"),
("g_coordinateUncertaintyInMeters", "double precision"),
("g_geodeticdatum", "varchar(50)"),
("g_higerGeography", "varchar(255)"),
("g_continent", "varchar(50)"),
("g_country", "varchar(50)"),
("g_countryCode", "varchar(50)"),
("g_stateProvince", "varchar(50)"),
("g_gbifid", "varchar(255)"),
("g_protocol", "varchar(255)"),
("g_identifier", "varchar(50)"),
("g_recordedby", "varchar(255)"),
("g_identificationid", "varchar(255)"),
("g_identifiers", "text"),
("g_dateidentified", "text"),
("g_modified", "text"),
("g_institutioncode", "varchar(50)"),
("g_lastinterpreted", "text"),
("g_lastparsed", "text"),
("g_references", "varchar(255)"),
("g_relations", "text"),
("g_catalognumber", "varchar(50)"),
("g_occurrencedetails", "text"),
("g_datasetkey", "varchar(50)"),
("g_datasetname", "varchar(255)"),
("g_collectioncode", "varchar(50)"),
("g_rights", "varchar(255)"),
("g_rightsholder", "varchar(255)"),
("g_license", "varchar(50)"),
("g_publishingorgkey", "varchar(50)"),
("g_publishingcountry", "varchar(50)"),
("g_lastcrawled", "text"),
("g_specificepithet", "varchar(50)"),
("g_facts", "text"),
("g_issues", "text"),
("g_extensions", "text"),
("g_language", "varchar(50)"),
]
# maybe no longer required in Python3
set_output_encoding()
# Set temporal filter if requested by user
# Initialize eventDate filter
eventDate = None
# Check if date from is compatible (ISO compliant)
if date_from:
try:
parse(date_from)
except:
grass.fatal("Invalid invalid start date provided")
if date_from and not date_to:
eventDate = "{}".format(date_from)
# Check if date to is compatible (ISO compliant)
if date_to:
try:
parse(date_to)
except:
grass.fatal("Invalid invalid end date provided")
# Check if date to is after date_from
if parse(date_from) < parse(date_to):
eventDate = "{},{}".format(date_from, date_to)
else:
grass.fatal(
"Invalid date range: End date has to be after start date!")
# Set filter on basisOfRecord if requested by user
if basisofrecord == "ALL":
basisOfRecord = None
else:
basisOfRecord = basisofrecord
# Allow also occurrences with spatial issues if requested by user
hasGeospatialIssue = False
if hasGeoIssue:
hasGeospatialIssue = True
# Allow also occurrences without coordinates if requested by user
hasCoordinate = True
if allow_no_geom:
hasCoordinate = False
# Set reprojection parameters
# Set target projection of current LOCATION
proj_info = grass.parse_command("g.proj", flags="g")
target_crs = grass.read_command("g.proj", flags="fj").rstrip()
target = osr.SpatialReference()
# Prefer EPSG CRS definitions
if proj_info["epsg"]:
target.ImportFromEPSG(int(proj_info["epsg"]))
else:
target.ImportFromProj4(target_crs)
# GDAL >= 3 swaps x and y axis, see: github.com/gdal/issues/1546
if int(gdal_version[0]) >= 3:
target.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
if target_crs == "XY location (unprojected)":
grass.fatal("Sorry, XY locations are not supported!")
# Set source projection from GBIF
source = osr.SpatialReference()
source.ImportFromEPSG(4326)
# GDAL >= 3 swaps x and y axis, see: github.com/gdal/issues/1546
if int(gdal_version[0]) >= 3:
source.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER)
if target_crs not in latlon_crs:
transform = osr.CoordinateTransformation(source, target)
reverse_transform = osr.CoordinateTransformation(target, source)
# Generate WKT polygon to use for spatial filtering if requested
if mask:
if len(mask.split("@")) == 2:
m = VectorTopo(mask.split("@")[0], mapset=mask.split("@")[1])
else:
m = VectorTopo(mask)
if not m.exist():
grass.fatal("Could not find vector map <{}>".format(mask))
m.open("r")
if not m.is_open():
grass.fatal("Could not open vector map <{}>".format(mask))
# Use map Bbox as spatial filter if map contains <> 1 area
if m.number_of("areas") == 1:
region_pol = [area.to_wkt() for area in m.viter("areas")][0]
else:
bbox = (str(m.bbox()).replace("Bbox(", "").replace(
" ", "").rstrip(")").split(","))
region_pol = "POLYGON (({0} {1}, {0} {3}, {2} {3}, {2} {1}, {0} {1}))".format(
bbox[2], bbox[0], bbox[3], bbox[1])
m.close()
else:
# Do not limit import spatially if LOCATION is able to take global data
if no_region_limit:
if target_crs not in latlon_crs:
grass.fatal("Import of data from outside the current region is"
"only supported in a WGS84 location!")
region_pol = None
else:
# Limit import spatially to current region
# if LOCATION is !NOT! able to take global data
# to avoid pprojection ERRORS
region = grass.parse_command("g.region", flags="g")
region_pol = "POLYGON (({0} {1},{0} {3},{2} {3},{2} {1},{0} {1}))".format(
region["e"], region["n"], region["w"], region["s"])
# Do not reproject in latlon LOCATIONS
if target_crs not in latlon_crs:
pol = ogr.CreateGeometryFromWkt(region_pol)
pol.Transform(reverse_transform)
pol = pol.ExportToWkt()
else:
pol = region_pol
# Create output map if not output maps for each species are requested
if (not species_maps and not print_species and not print_species_shell
and not print_occ_number and not print_species_table):
mapname = output
new = Vector(mapname)
new.open("w", tab_name=mapname, tab_cols=cols)
cat = 1
# Import data for each species
for s in taxa_list:
# Get the taxon key if not the taxon key is provided as input
try:
key = int(s)
except:
try:
species_match = species.name_backbone(s,
rank=rank,
strict=False,
verbose=True)
key = species_match["usageKey"]
except:
grass.error(
"Data request for taxon {} failed. Are you online?".format(
s))
continue
# Return matching taxon and alternatives and exit
if print_species:
print("Matching taxon for {} is:".format(s))
print("{} {}".format(species_match["scientificName"],
species_match["status"]))
if "alternatives" in list(species_match.keys()):
print("Alternative matches might be: {}".format(s))
for m in species_match["alternatives"]:
print("{} {}".format(m["scientificName"], m["status"]))
else:
print("No alternatives found for the given taxon")
continue
if print_species_shell:
print("match={}".format(species_match["scientificName"]))
if "alternatives" in list(species_match.keys()):
alternatives = []
for m in species_match["alternatives"]:
alternatives.append(m["scientificName"])
print("alternatives={}".format(",".join(alternatives)))
continue
if print_species_table:
if "alternatives" in list(species_match.keys()):
if len(species_match["alternatives"]) == 0:
print("{0}|{1}|{2}|".format(
s, key, species_match["scientificName"]))
else:
alternatives = []
for m in species_match["alternatives"]:
alternatives.append(m["scientificName"])
print("{0}|{1}|{2}|{3}".format(
s,
key,
species_match["scientificName"],
",".join(alternatives),
))
continue
try:
returns_n = occurrences.search(
taxonKey=key,
hasGeospatialIssue=hasGeospatialIssue,
hasCoordinate=hasCoordinate,
institutionCode=institutionCode,
basisOfRecord=basisOfRecord,
recordedBy=recordedby,
eventDate=eventDate,
continent=continent,
country=country,
geometry=pol,
limit=1,
)["count"]
except:
grass.error(
"Data request for taxon {} faild. Are you online?".format(s))
returns_n = 0
# Exit if search does not give a return
# Print only number of returns for the given search and exit
if print_occ_number:
print("Found {0} occurrences for taxon {1}...".format(
returns_n, s))
continue
elif returns_n <= 0:
grass.warning(
"No occurrences for current search for taxon {0}...".format(s))
continue
elif returns_n >= 200000:
grass.warning(
"Your search for {1} returns {0} records.\n"
"Unfortunately, the GBIF search API is limited to 200,000 records per request.\n"
"The download will be incomplete. Please consider to split up your search."
.format(returns_n, s))
# Get the number of chunks to download
chunks = int(math.ceil(returns_n / float(chunk_size)))
grass.verbose("Downloading {0} occurrences for taxon {1}...".format(
returns_n, s))
# Create a map for each species if requested using map name as suffix
if species_maps:
mapname = "{}_{}".format(s.replace(" ", "_"), output)
new = Vector(mapname)
new.open("w", tab_name=mapname, tab_cols=cols)
cat = 0
# Download the data from GBIF
for c in range(chunks):
# Define offset
offset = c * chunk_size
# Adjust chunk_size to the hard limit of 200,000 records in GBIF API
# if necessary
if offset + chunk_size >= 200000:
chunk_size = 200000 - offset
# Get the returns for the next chunk
returns = occurrences.search(
taxonKey=key,
hasGeospatialIssue=hasGeospatialIssue,
hasCoordinate=hasCoordinate,
institutionCode=institutionCode,
basisOfRecord=basisOfRecord,
recordedBy=recordedby,
eventDate=eventDate,
continent=continent,
country=country,
geometry=pol,
limit=chunk_size,
offset=offset,
)
# Write the returned data to map and attribute table
for res in returns["results"]:
if target_crs not in latlon_crs:
point = ogr.CreateGeometryFromWkt("POINT ({} {})".format(
res["decimalLongitude"], res["decimalLatitude"]))
point.Transform(transform)
x = point.GetX()
y = point.GetY()
else:
x = res["decimalLongitude"]
y = res["decimalLatitude"]
point = Point(x, y)
for k in dwc_keys:
if k not in list(res.keys()):
res.update({k: None})
cat = cat + 1
new.write(
point,
cat=cat,
attrs=(
"{}".format(s),
res["key"],
res["taxonRank"],
res["taxonKey"],
res["taxonID"],
res["scientificName"],
res["species"],
res["speciesKey"],
res["genericName"],
res["genus"],
res["genusKey"],
res["family"],
res["familyKey"],
res["order"],
res["orderKey"],
res["class"],
res["classKey"],
res["phylum"],
res["phylumKey"],
res["kingdom"],
res["kingdomKey"],
"{}".format(res["eventDate"])
if res["eventDate"] else None,
"{}".format(res["verbatimEventDate"])
if res["verbatimEventDate"] else None,
res["startDayOfYear"],
res["endDayOfYear"],
res["year"],
res["month"],
res["day"],
res["occurrenceID"],
res["occurrenceStatus"],
res["occurrenceRemarks"],
res["Habitat"],
res["basisOfRecord"],
res["preparations"],
res["sex"],
res["type"],
res["locality"],
res["verbatimLocality"],
res["decimalLongitude"],
res["decimalLatitude"],
res["coordinateUncertaintyInMeters"],
res["geodeticDatum"],
res["higerGeography"],
res["continent"],
res["country"],
res["countryCode"],
res["stateProvince"],
res["gbifID"],
res["protocol"],
res["identifier"],
res["recordedBy"],
res["identificationID"],
",".join(res["identifiers"]),
"{}".format(res["dateIdentified"])
if res["dateIdentified"] else None,
"{}".format(res["modified"])
if res["modified"] else None,
res["institutionCode"],
"{}".format(res["lastInterpreted"])
if res["lastInterpreted"] else None,
"{}".format(res["lastParsed"])
if res["lastParsed"] else None,
res["references"],
",".join(res["relations"]),
res["catalogNumber"],
"{}".format(res["occurrenceDetails"])
if res["occurrenceDetails"] else None,
res["datasetKey"],
res["datasetName"],
res["collectionCode"],
res["rights"],
res["rightsHolder"],
res["license"],
res["publishingOrgKey"],
res["publishingCountry"],
"{}".format(res["lastCrawled"])
if res["lastCrawled"] else None,
res["specificEpithet"],
",".join(res["facts"]),
",".join(res["issues"]),
",".join(res["extensions"]),
res["language"],
),
)
cat = cat + 1
# Close the current map if a map for each species is requested
if species_maps:
new.table.conn.commit()
new.close()
if not no_topo:
grass.run_command("v.build", map=mapname, option="build")
# Write history to map
grass.vector_history(mapname)
# Close the output map if not a map for each species is requested
if (not species_maps and not print_species and not print_species_shell
and not print_occ_number and not print_species_table):
new.table.conn.commit()
new.close()
if not no_topo:
grass.run_command("v.build", map=mapname, option="build")
# Write history to map
grass.vector_history(mapname)
def main(opt, flg):
# import functions which depend on sklearn only after parser run
from ml_functions import (
balance,
explorer_clsfiers,
run_classifier,
optimize_training,
explore_SVC,
plot_grid,
)
from features import importances, tocsv
msgr = get_msgr()
indexes = None
vect = opt["vector"]
vtraining = opt["vtraining"] if opt["vtraining"] else None
scaler, decmp = None, None
vlayer = opt["vlayer"] if opt["vlayer"] else vect + "_stats"
tlayer = opt["tlayer"] if opt["tlayer"] else vect + "_training"
rlayer = opt["rlayer"] if opt["rlayer"] else vect + "_results"
labels = extract_classes(vtraining, 1)
pprint(labels)
if opt["scalar"]:
scapar = opt["scalar"].split(",")
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler(with_mean="with_mean" in scapar,
with_std="with_std" in scapar)
if opt["decomposition"]:
dec, params = (opt["decomposition"].split("|") if "|"
in opt["decomposition"] else (opt["decomposition"], ""))
kwargs = ({k: v
for k, v in (p.split("=")
for p in params.split(","))} if params else {})
load_decompositions()
decmp = DECMP[dec](**kwargs)
# if training extract training
if vtraining and flg["e"]:
msgr.message("Extract training from: <%s> to <%s>." %
(vtraining, vect))
extract_training(vect, vtraining, tlayer)
flg["n"] = True
if flg["n"]:
msgr.message("Save arrays to npy files.")
save2npy(
vect,
vlayer,
tlayer,
fcats=opt["npy_cats"],
fcols=opt["npy_cols"],
fdata=opt["npy_data"],
findx=opt["npy_index"],
fclss=opt["npy_tclasses"],
ftdata=opt["npy_tdata"],
)
# define the classifiers to use/test
if opt["pyclassifiers"] and opt["pyvar"]:
# import classifiers to use
mycls = SourceFileLoader("mycls", opt["pyclassifiers"]).load_module()
classifiers = getattr(mycls, opt["pyvar"])
else:
from ml_classifiers import CLASSIFIERS
classifiers = CLASSIFIERS
# Append the SVC classifier
if opt["svc_c"] and opt["svc_gamma"]:
from sklearn.svm import SVC
svc = {
"name": "SVC",
"classifier": SVC,
"kwargs": {
"C": float(opt["svc_c"]),
"gamma": float(opt["svc_gamma"]),
"kernel": opt["svc_kernel"],
},
}
classifiers.append(svc)
# extract classifiers from pyindx
if opt["pyindx"]:
indexes = [i for i in get_indexes(opt["pyindx"])]
classifiers = [classifiers[i] for i in indexes]
num = int(opt["n_training"]) if opt["n_training"] else None
# load fron npy files
Xt = np.load(opt["npy_tdata"])
Yt = np.load(opt["npy_tclasses"])
cols = np.load(opt["npy_cols"])
# Define rules to substitute NaN, Inf, posInf, negInf values
rules = {}
for key in ("nan", "inf", "neginf", "posinf"):
if opt[key]:
rules[key] = get_rules(opt[key])
pprint(rules)
# Substitute (skip cat column)
Xt, rules_vals = substitute(Xt, rules, cols[1:])
Xtoriginal = Xt
# scale the data
if scaler:
msgr.message("Scaling the training data set.")
scaler.fit(Xt, Yt)
Xt = scaler.transform(Xt)
# decompose data
if decmp:
msgr.message("Decomposing the training data set.")
decmp.fit(Xt)
Xt = decmp.transform(Xt)
# Feature importances with forests of trees
if flg["f"]:
np.save("training_transformed.npy", Xt)
importances(
Xt,
Yt,
cols[1:],
csv=opt["imp_csv"],
img=opt["imp_fig"],
# default parameters to save the matplotlib figure
**dict(dpi=300, transparent=False, bbox_inches="tight"),
)
# optimize the training set
if flg["o"]:
ind_optimize = int(
opt["pyindx_optimize"]) if opt["pyindx_optimize"] else 0
cls = classifiers[ind_optimize]
msgr.message("Find the optimum training set.")
best, Xbt, Ybt = optimize_training(
cls,
Xt,
Yt,
labels, # {v: k for k, v in labels.items()},
scaler,
decmp,
num=num,
maxiterations=1000,
)
msg = " - save the optimum training data set to: %s."
msgr.message(msg % opt["npy_btdata"])
np.save(opt["npy_btdata"], Xbt)
msg = " - save the optimum training classes set to: %s."
msgr.message(msg % opt["npy_btclasses"])
np.save(opt["npy_btclasses"], Ybt)
# balance the data
if flg["b"]:
msg = "Balancing the training data set, each class have <%d> samples."
msgr.message(msg % num)
Xbt, Ybt = balance(Xt, Yt, num)
else:
if not flg["o"]:
Xbt = (np.load(opt["npy_btdata"])
if os.path.isfile(opt["npy_btdata"]) else Xt)
Ybt = (np.load(opt["npy_btclasses"])
if os.path.isfile(opt["npy_btclasses"]) else Yt)
# scale the data
if scaler:
msgr.message("Scaling the training data set.")
scaler.fit(Xbt, Ybt)
Xt = scaler.transform(Xt)
Xbt = scaler.transform(Xbt)
if flg["d"]:
C_range = [float(c) for c in opt["svc_c_range"].split(",") if c]
gamma_range = [
float(g) for g in opt["svc_gamma_range"].split(",") if g
]
kernel_range = [
str(s) for s in opt["svc_kernel_range"].split(",") if s
]
poly_range = [int(i) for i in opt["svc_poly_range"].split(",") if i]
allkwargs = dict(C=C_range,
gamma=gamma_range,
kernel=kernel_range,
degree=poly_range)
kwargs = {}
for k in allkwargs:
if allkwargs[k]:
kwargs[k] = allkwargs[k]
msgr.message("Exploring the SVC domain.")
grid = explore_SVC(Xbt,
Ybt,
n_folds=5,
n_jobs=int(opt["svc_n_jobs"]),
**kwargs)
import pickle
krnlstr = "_".join(s for s in opt["svc_kernel_range"].split(",") if s)
pkl = open("grid%s.pkl" % krnlstr, "w")
pickle.dump(grid, pkl)
pkl.close()
# pkl = open('grid.pkl', 'r')
# grid = pickle.load(pkl)
# pkl.close()
plot_grid(grid, save=opt["svc_img"])
# test the accuracy of different classifiers
if flg["t"]:
# test different classifiers
msgr.message("Exploring different classifiers.")
msgr.message("cls_id cls_name mean max min std")
res = explorer_clsfiers(
classifiers,
Xt,
Yt,
labels=labels,
indexes=indexes,
n_folds=5,
bv=flg["v"],
extra=flg["x"],
)
# TODO: sort(order=...) is working only in the terminal, why?
# res.sort(order='mean')
with open(opt["csv_test_cls"], "w") as csv:
csv.write(tocsv(res))
if flg["c"]:
# classify
data = np.load(opt["npy_data"])
indx = np.load(opt["npy_index"])
# Substitute using column values
data, dummy = substitute(data, rules, cols[1:])
Xt = data[indx]
if scaler:
msgr.message("Scaling the training data set.")
scaler.fit(Xt, Yt)
Xt = scaler.transform(Xt)
msgr.message("Scaling the whole data set.")
data = scaler.transform(data)
if decmp:
msgr.message("Decomposing the training data set.")
decmp.fit(Xt)
Xt = decmp.transform(Xt)
msgr.message("Decompose the whole data set.")
data = decmp.transform(data)
cats = np.load(opt["npy_cats"])
np.save("data_filled_scaled.npy", data)
tcols = []
for cls in classifiers:
report = (open(opt["report_class"], "w")
if opt["report_class"] else sys.stdout)
run_classifier(cls, Xt, Yt, Xt, Yt, labels, data, report=report)
tcols.append((cls["name"], "INTEGER"))
import pickle
with open("classification_results.pkl", "w") as res:
pickle.dump(classifiers, res)
# classifiers = pickle.load(res)
msgr.message("Export the results to layer: <%s>" % str(rlayer))
export_results(
vect,
classifiers,
cats,
rlayer,
vtraining,
tcols,
overwrite(),
pkl="res.pkl",
append=flg["a"],
)
# res.close()
if flg["r"]:
rules = ("\n".join([
"%d %s" % (k, v) for k, v in get_colors(vtraining).items()
]) if vtraining else None)
msgr.message("Export the layer with results to raster")
with Vector(vect, mode="r") as vct:
tab = vct.dblinks.by_name(rlayer).table()
rasters = [c for c in tab.columns]
rasters.remove(tab.key)
v2rst = Module("v.to.rast")
rclrs = Module("r.colors")
for rst in rasters:
v2rst(
input=vect,
layer=rlayer,
type="area",
use="attr",
attrcolumn=rst.encode(),
output=(opt["rst_names"] % rst).encode(),
memory=1000,
overwrite=overwrite(),
)
if rules:
rclrs(map=rst.encode(), rules="-", stdin_=rules)
def get_colors(vtraining):
vect, mset = vtraining.split('@') if '@' in vtraining else (vtraining, '')
with Vector(vect, mapset=mset, mode='r') as vct:
cur = vct.table.execute('SELECT cat, color FROM %s;' % vct.name)
return dict([c for c in cur.fetchall()])
area0s_list = list(area0s)
sorted_cats_by_area = [cats_list for (area0s_list,cats_list) in sorted(zip(area0s_list,cats_list))]
from grass.pygrass.vector import Vector, VectorTopo
streams = Vector('streams')
streamsTopo = VectorTopo('streams')
# to get list of lines:
# streamsTopo.open(mode='r')
# streamsTopo.next()
streamsTopo.open()
"""
# THIS IS HOW YOU UPDATE A TABLE!!!!
v.db_addcolumn(map='streams', columns='stream_number_ascending int')
#streams.table.columns.add('stream_number_ascending','int')
streams = Vector('streams')
streams.open('rw')
cur = streams.table.conn.cursor()
for i in range(len(sorted_new_river_numbers)):
print i
cur.execute("update streams set stream_number_ascending=" +
str(river_numbers[i]) + " where cat=" + str(cats_list[i]))
streams.table.conn.commit()
streams.build()
# Should overwrite river_numbers, eventually
colNames = np.array(vector_db_select('streams')['columns'])
colValues = np.array(vector_db_select('streams')['values'].values())
river_numbers = colValues[:, colNames == 'stream_number_ascending'].astype(
int).squeeze()
# Could also just use our river_numbers vector from before
def main(opt, flg):
# import functions which depend on sklearn only after parser run
from ml_functions import (balance, explorer_clsfiers, run_classifier,
optimize_training, explore_SVC, plot_grid)
from features import importances, tocsv
msgr = get_msgr()
indexes = None
vect = opt['vector']
vtraining = opt['vtraining'] if opt['vtraining'] else None
scaler, decmp = None, None
vlayer = opt['vlayer'] if opt['vlayer'] else vect + '_stats'
tlayer = opt['tlayer'] if opt['tlayer'] else vect + '_training'
rlayer = opt['rlayer'] if opt['rlayer'] else vect + '_results'
labels = extract_classes(vtraining, 1)
pprint(labels)
if opt['scalar']:
scapar = opt['scalar'].split(',')
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler(with_mean='with_mean' in scapar,
with_std='with_std' in scapar)
if opt['decomposition']:
dec, params = (opt['decomposition'].split('|')
if '|' in opt['decomposition']
else (opt['decomposition'], ''))
kwargs = ({k: v for k, v in (p.split('=') for p in params.split(','))}
if params else {})
load_decompositions()
decmp = DECMP[dec](**kwargs)
# if training extract training
if vtraining and flg['e']:
msgr.message("Extract training from: <%s> to <%s>." % (vtraining, vect))
extract_training(vect, vtraining, tlayer)
flg['n'] = True
if flg['n']:
msgr.message("Save arrays to npy files.")
save2npy(vect, vlayer, tlayer,
fcats=opt['npy_cats'], fcols=opt['npy_cols'],
fdata=opt['npy_data'], findx=opt['npy_index'],
fclss=opt['npy_tclasses'], ftdata=opt['npy_tdata'])
# define the classifiers to use/test
if opt['pyclassifiers'] and opt['pyvar']:
# import classifiers to use
mycls = imp.load_source("mycls", opt['pyclassifiers'])
classifiers = getattr(mycls, opt['pyvar'])
else:
from ml_classifiers import CLASSIFIERS
classifiers = CLASSIFIERS
# Append the SVC classifier
if opt['svc_c'] and opt['svc_gamma']:
from sklearn.svm import SVC
svc = {'name': 'SVC', 'classifier': SVC,
'kwargs': {'C': float(opt['svc_c']),
'gamma': float(opt['svc_gamma']),
'kernel': opt['svc_kernel']}}
classifiers.append(svc)
# extract classifiers from pyindx
if opt['pyindx']:
indexes = [i for i in get_indexes(opt['pyindx'])]
classifiers = [classifiers[i] for i in indexes]
num = int(opt['n_training']) if opt['n_training'] else None
# load fron npy files
Xt = np.load(opt['npy_tdata'])
Yt = np.load(opt['npy_tclasses'])
cols = np.load(opt['npy_cols'])
# Define rules to substitute NaN, Inf, posInf, negInf values
rules = {}
for key in ('nan', 'inf', 'neginf', 'posinf'):
if opt[key]:
rules[key] = get_rules(opt[key])
pprint(rules)
# Substitute (skip cat column)
Xt, rules_vals = substitute(Xt, rules, cols[1:])
Xtoriginal = Xt
# scale the data
if scaler:
msgr.message("Scaling the training data set.")
scaler.fit(Xt, Yt)
Xt = scaler.transform(Xt)
# decompose data
if decmp:
msgr.message("Decomposing the training data set.")
decmp.fit(Xt)
Xt = decmp.transform(Xt)
# Feature importances with forests of trees
if flg['f']:
np.save('training_transformed.npy', Xt)
importances(Xt, Yt, cols[1:],
csv=opt['imp_csv'], img=opt['imp_fig'],
# default parameters to save the matplotlib figure
**dict(dpi=300, transparent=False, bbox_inches='tight'))
# optimize the training set
if flg['o']:
ind_optimize = (int(opt['pyindx_optimize']) if opt['pyindx_optimize']
else 0)
cls = classifiers[ind_optimize]
msgr.message("Find the optimum training set.")
best, Xbt, Ybt = optimize_training(cls, Xt, Yt,
labels, #{v: k for k, v in labels.items()},
scaler, decmp,
num=num, maxiterations=1000)
msg = " - save the optimum training data set to: %s."
msgr.message(msg % opt['npy_btdata'])
np.save(opt['npy_btdata'], Xbt)
msg = " - save the optimum training classes set to: %s."
msgr.message(msg % opt['npy_btclasses'])
np.save(opt['npy_btclasses'], Ybt)
# balance the data
if flg['b']:
msg = "Balancing the training data set, each class have <%d> samples."
msgr.message(msg % num)
Xbt, Ybt = balance(Xt, Yt, num)
else:
if not flg['o']:
Xbt = (np.load(opt['npy_btdata'])
if os.path.isfile(opt['npy_btdata']) else Xt)
Ybt = (np.load(opt['npy_btclasses'])
if os.path.isfile(opt['npy_btclasses']) else Yt)
# scale the data
if scaler:
msgr.message("Scaling the training data set.")
scaler.fit(Xbt, Ybt)
Xt = scaler.transform(Xt)
Xbt = scaler.transform(Xbt)
if flg['d']:
C_range = [float(c) for c in opt['svc_c_range'].split(',') if c]
gamma_range = [float(g) for g in opt['svc_gamma_range'].split(',') if g]
kernel_range = [str(s) for s in opt['svc_kernel_range'].split(',') if s]
poly_range = [int(i) for i in opt['svc_poly_range'].split(',') if i]
allkwargs = dict(C=C_range, gamma=gamma_range,
kernel=kernel_range, degree=poly_range)
kwargs = {}
for k in allkwargs:
if allkwargs[k]:
kwargs[k] = allkwargs[k]
msgr.message("Exploring the SVC domain.")
grid = explore_SVC(Xbt, Ybt, n_folds=5, n_jobs=int(opt['svc_n_jobs']),
**kwargs)
import pickle
krnlstr = '_'.join(s for s in opt['svc_kernel_range'].split(',') if s)
pkl = open('grid%s.pkl' % krnlstr, 'w')
pickle.dump(grid, pkl)
pkl.close()
# pkl = open('grid.pkl', 'r')
# grid = pickle.load(pkl)
# pkl.close()
plot_grid(grid, save=opt['svc_img'])
# test the accuracy of different classifiers
if flg['t']:
# test different classifiers
msgr.message("Exploring different classifiers.")
msgr.message("cls_id cls_name mean max min std")
res = explorer_clsfiers(classifiers, Xt, Yt, labels=labels,
indexes=indexes, n_folds=5,
bv=flg['v'], extra=flg['x'])
# TODO: sort(order=...) is working only in the terminal, why?
#res.sort(order='mean')
with open(opt['csv_test_cls'], 'w') as csv:
csv.write(tocsv(res))
if flg['c']:
# classify
data = np.load(opt['npy_data'])
indx = np.load(opt['npy_index'])
# Substitute using column values
data, dummy = substitute(data, rules, cols[1:])
Xt = data[indx]
if scaler:
msgr.message("Scaling the training data set.")
scaler.fit(Xt, Yt)
Xt = scaler.transform(Xt)
msgr.message("Scaling the whole data set.")
data = scaler.transform(data)
if decmp:
msgr.message("Decomposing the training data set.")
decmp.fit(Xt)
Xt = decmp.transform(Xt)
msgr.message("Decompose the whole data set.")
data = decmp.transform(data)
cats = np.load(opt['npy_cats'])
np.save('data_filled_scaled.npy', data)
tcols = []
for cls in classifiers:
report = (open(opt['report_class'], "w")
if opt['report_class'] else sys.stdout)
run_classifier(cls, Xt, Yt, Xt, Yt, labels, data,
report=report)
tcols.append((cls['name'], 'INTEGER'))
import pickle
with open('classification_results.pkl', 'w') as res:
pickle.dump(classifiers, res)
#classifiers = pickle.load(res)
msgr.message("Export the results to layer: <%s>" % str(rlayer))
export_results(vect, classifiers, cats, rlayer, vtraining, tcols,
overwrite(), pkl='res.pkl', append=flg['a'])
# res.close()
if flg['r']:
rules = ('\n'.join(['%d %s' % (k, v)
for k, v in get_colors(vtraining).items()])
if vtraining else None)
msgr.message("Export the layer with results to raster")
with Vector(vect, mode='r') as vct:
tab = vct.dblinks.by_name(rlayer).table()
rasters = [c for c in tab.columns]
rasters.remove(tab.key)
v2rst = Module('v.to.rast')
rclrs = Module('r.colors')
for rst in rasters:
v2rst(input=vect, layer=rlayer, type='area',
use='attr', attrcolumn=rst.encode(),
output=(opt['rst_names'] % rst).encode(),
memory=1000, overwrite=overwrite())
if rules:
rclrs(map=rst.encode(), rules='-', stdin_=rules)
#!/usr/bin/env python3
from grass.pygrass.raster import RasterRow
from grass.pygrass.vector import Vector
from grass.pygrass.gis.region import Region
from grass.pygrass.utils import coor2pixel
name = 'dmt@PERMANENT'
reg = Region()
reg.from_rast(name)
reg.set_current()
dmt = RasterRow(name)
dmt.open('r')
obce = Vector('obce_bod')
obce.open('r')
for o in obce:
x, y = coor2pixel(o.coords(), region)
value = dmt[int(x)][int(y)]
print ('{:40}: {:.0f}'.format(o.attrs['nazev'], value))
obce.close()
dmt.close()