def main():
options, flags = gcore.parser()
raster = options['raster']
raster_3d = options['raster_3d']
vector = options['vector']
sep = separator(options['separator'])
perform_pre_checks = not flags['s']
dry_run = flags['d']
if perform_pre_checks:
if raster:
check_file(raster, 'raster', sep=sep)
if raster_3d:
check_file(raster_3d, 'raster_3d', sep=sep)
if vector:
check_file(vector, 'vector', sep=sep)
if dry_run:
gcore.message(_("Checks successful"))
return
if raster:
rename_from_file(filename=raster, map_type='raster', sep=sep,
safe_input=perform_pre_checks)
if raster_3d:
rename_from_file(filename=raster_3d, map_type='raster_3d', sep=sep,
safe_input=perform_pre_checks)
if vector:
rename_from_file(filename=vector, map_type='vector', sep=sep,
safe_input=perform_pre_checks)
def main():
options, flags = gcore.parser()
raster = options["raster"]
raster_3d = options["raster_3d"]
vector = options["vector"]
sep = separator(options["separator"])
perform_pre_checks = not flags["s"]
dry_run = flags["d"]
if perform_pre_checks:
if raster:
check_file(raster, "raster", sep=sep)
if raster_3d:
check_file(raster_3d, "raster_3d", sep=sep)
if vector:
check_file(vector, "vector", sep=sep)
if dry_run:
gcore.message(_("Checks successful"))
return
if raster:
rename_from_file(filename=raster,
map_type="raster",
sep=sep,
safe_input=perform_pre_checks)
if raster_3d:
rename_from_file(
filename=raster_3d,
map_type="raster_3d",
sep=sep,
safe_input=perform_pre_checks,
)
if vector:
rename_from_file(filename=vector,
map_type="vector",
sep=sep,
safe_input=perform_pre_checks)
def main():
mapname = options["map"]
layer = options["layer"]
option = options["option"]
units = options["units"]
sort = options["sort"]
fs = separator(options["separator"])
nuldev = open(os.devnull, "w")
if not grass.find_file(mapname, "vector")["file"]:
grass.fatal(_("Vector map <%s> not found") % mapname)
if int(layer) in grass.vector_db(mapname):
colnames = grass.vector_columns(mapname,
layer,
getDict=False,
stderr=nuldev)
isConnection = True
else:
isConnection = False
colnames = ["cat"]
if option == "coor":
extracolnames = ["x", "y", "z"]
else:
extracolnames = [option]
if units == "percent":
unitsp = "meters"
elif units:
unitsp = units
else:
unitsp = None
# NOTE: we suppress -1 cat and 0 cat
if isConnection:
f = grass.vector_db(map=mapname)[int(layer)]
p = grass.pipe_command("v.db.select",
flags="e",
quiet=True,
map=mapname,
layer=layer)
records1 = []
catcol = -1
ncols = 0
for line in p.stdout:
cols = decode(line).rstrip("\r\n").split("|")
if catcol == -1:
ncols = len(cols)
for i in range(0, ncols):
if cols[i] == f["key"]:
catcol = i
break
if catcol == -1:
grass.fatal(
_("There is a table connected to input vector map '%s', but "
"there is no key column '%s'.") %
(mapname, f["key"]))
continue
if cols[catcol] == "-1" or cols[catcol] == "0":
continue
records1.append(cols[:catcol] + [int(cols[catcol])] +
cols[(catcol + 1):])
p.wait()
if p.returncode != 0:
sys.exit(1)
records1.sort(key=lambda r: r[catcol])
if len(records1) == 0:
try:
grass.fatal(
_("There is a table connected to input vector map '%s', but "
"there are no categories present in the key column '%s'. Consider using "
"v.to.db to correct this.") % (mapname, f["key"]))
except KeyError:
pass
# fetch the requested attribute sorted by cat:
p = grass.pipe_command(
"v.to.db",
flags="p",
quiet=True,
map=mapname,
option=option,
layer=layer,
units=unitsp,
)
records2 = []
for line in p.stdout:
fields = decode(line).rstrip("\r\n").split("|")
if fields[0] in ["cat", "-1", "0"]:
continue
records2.append([int(fields[0])] + fields[1:])
p.wait()
records2.sort()
# make pre-table
# len(records1) may not be the same as len(records2) because
# v.db.select can return attributes that are not linked to features.
records3 = []
for r2 in records2:
rec = list(filter(lambda r1: r1[catcol] == r2[0], records1))
if len(rec) > 0:
res = rec[0] + r2[1:]
elif flags["d"]:
res = [r2[0]] + [""] * (ncols - 1) + r2[1:]
else:
continue
records3.append(res)
else:
catcol = 0
records1 = []
p = grass.pipe_command("v.category",
inp=mapname,
layer=layer,
option="print")
for line in p.stdout:
field = int(decode(line).rstrip())
if field > 0:
records1.append(field)
p.wait()
records1.sort()
records1 = uniq(records1)
# make pre-table
p = grass.pipe_command(
"v.to.db",
flags="p",
quiet=True,
map=mapname,
option=option,
layer=layer,
units=unitsp,
)
records3 = []
for line in p.stdout:
fields = decode(line).rstrip("\r\n").split("|")
if fields[0] in ["cat", "-1", "0"]:
continue
records3.append([int(fields[0])] + fields[1:])
p.wait()
records3.sort()
# print table header
if not flags["c"]:
sys.stdout.write(fs.join(colnames + extracolnames) + "\n")
# make and print the table:
numcols = len(colnames) + len(extracolnames)
# calculate percents if requested
if units == "percent" and option != "coor":
# calculate total value
total = 0
for r in records3:
total += float(r[-1])
# calculate percentages
records4 = [float(r[-1]) * 100 / total for r in records3]
if type(records1[0]) == int:
records3 = [[r1] + [r4] for r1, r4 in zip(records1, records4)]
else:
records3 = [r1 + [r4] for r1, r4 in zip(records1, records4)]
# sort results
if sort:
if sort == "asc":
if option == "coor":
records3.sort(
key=lambda r: (float(r[-3]), float(r[-2]), float(r[-1])))
else:
records3.sort(key=lambda r: float(r[-1]))
else:
if option == "coor":
records3.sort(
key=lambda r: (float(r[-3]), float(r[-2]), float(r[-1])),
reverse=True,
)
else:
records3.sort(key=lambda r: float(r[-1]), reverse=True)
for r in records3:
sys.stdout.write(fs.join(map(str, r)) + "\n")
def main():
vinput = options['input']
columns = options['columns'].split(',')
binary = options['developed_column']
level = options['subregions_column']
sep = gutils.separator(options['separator'])
minim = int(options['min_variables'])
dredge = flags['d']
if options['max_variables']:
maxv = int(options['max_variables'])
else:
maxv = len(columns)
if dredge and minim > maxv:
gscript.fatal(_("Minimum number of predictor variables is larger than maximum number"))
global TMP_CSV, TMP_RSCRIPT, TMP_POT
TMP_CSV = gscript.tempfile(create=False) + '.csv'
TMP_RSCRIPT = gscript.tempfile()
include_level = True
distinct = gscript.read_command('v.db.select', flags='c', map=vinput,
columns="distinct {level}".format(level=level)).strip()
if len(distinct.splitlines()) <= 1:
include_level = False
single_level = distinct.splitlines()[0]
with open(TMP_RSCRIPT, 'w') as f:
f.write(rscript)
TMP_POT = gscript.tempfile(create=False) + '_potential.csv'
columns += [binary]
if include_level:
columns += [level]
where = "{c} IS NOT NULL".format(c=columns[0])
for c in columns[1:]:
where += " AND {c} IS NOT NULL".format(c=c)
gscript.run_command('v.db.select', map=vinput, columns=columns, separator='comma', where=where, file=TMP_CSV)
if dredge:
gscript.info(_("Running automatic model selection ..."))
else:
gscript.info(_("Computing model..."))
cmd = ['Rscript', TMP_RSCRIPT, '-i', TMP_CSV, '-r', binary,
'-m', str(minim), '-x', str(maxv), '-o', TMP_POT, '-d', 'TRUE' if dredge else 'FALSE']
if include_level:
cmd += ['-l', level]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
gscript.warning(gscript.decode(stderr))
if p.returncode != 0:
gscript.warning(gscript.decode(stderr))
gscript.fatal(_("Running R script failed, check messages above"))
gscript.info(_("Best model summary:"))
gscript.info("-------------------------")
gscript.message(gscript.decode(stdout))
with open(TMP_POT, 'r') as fin, open(options['output'], 'w') as fout:
i = 0
for line in fin.readlines():
row = line.strip().split('\t')
row = [each.strip('"') for each in row]
if i == 0:
row[0] = "ID"
row[1] = "Intercept"
if i == 1 and not include_level:
row[0] = single_level
fout.write(sep.join(row))
fout.write('\n')
i += 1
def main():
global tmp
fs = separator(options['separator'])
threeD = flags['z']
prog = 'v.in.lines'
if threeD:
do3D = 'z'
else:
do3D = ''
tmp = grass.tempfile()
#### set up input file
if options['input'] == '-':
infile = None
inf = sys.stdin
else:
infile = options['input']
if not os.path.exists(infile):
grass.fatal(_("Unable to read input file <%s>") % infile)
grass.debug("input file=[%s]" % infile)
if not infile:
# read from stdin and write to tmpfile (v.in.mapgen wants a real file)
outf = file(tmp, 'w')
for line in inf:
if len(line.lstrip()) == 0 or line[0] == '#':
continue
outf.write(line.replace(fs, ' '))
outf.close()
runfile = tmp
else:
# read from a real file
if fs == ' ':
runfile = infile
else:
inf = file(infile)
outf = file(tmp, 'w')
for line in inf:
if len(line.lstrip()) == 0 or line[0] == '#':
continue
outf.write(line.replace(fs, ' '))
inf.close()
outf.close()
runfile = tmp
##### check that there are at least two columns (three if -z is given)
inf = file(runfile)
for line in inf:
if len(line.lstrip()) == 0 or line[0] == '#':
continue
numcols = len(line.split())
break
inf.close()
if (do3D and numcols < 3) or (not do3D and numcols < 2):
grass.fatal(_("Not enough data columns. (incorrect fs setting?)"))
grass.run_command('v.in.mapgen', flags = 'f' + do3D,
input = runfile, output = options['output'])
def main():
coords = options["coordinates"]
input = options["input"]
output = options["output"]
fs = options["separator"]
proj_in = options["proj_in"]
proj_out = options["proj_out"]
ll_in = flags["i"]
ll_out = flags["o"]
decimal = flags["d"]
copy_input = flags["e"]
include_header = flags["c"]
# check for cs2cs
if not gcore.find_program("cs2cs"):
gcore.fatal(
_("cs2cs program not found, install PROJ first: \
https://proj.org"))
# parse field separator
# FIXME: input_x,y needs to split on multiple whitespace between them
if fs == ",":
ifs = ofs = ","
else:
try:
ifs, ofs = fs.split(",")
except ValueError:
ifs = ofs = fs
ifs = separator(ifs)
ofs = separator(ofs)
# set up projection params
s = gcore.read_command("g.proj", flags="j")
kv = parse_key_val(s)
if "XY location" in kv["+proj"] and (ll_in or ll_out):
gcore.fatal(_("Unable to project to or from a XY location"))
in_proj = None
if ll_in:
in_proj = "+proj=longlat +datum=WGS84"
gcore.verbose(
"Assuming LL WGS84 as input, current projection as output ")
if ll_out:
in_proj = gcore.read_command("g.proj", flags="jf")
if proj_in:
if "+" in proj_in:
in_proj = proj_in
else:
gcore.fatal(_("Invalid PROJ.4 input specification"))
if not in_proj:
gcore.verbose("Assuming current location as input")
in_proj = gcore.read_command("g.proj", flags="jf")
in_proj = in_proj.strip()
gcore.verbose("Input parameters: '%s'" % in_proj)
out_proj = None
if ll_out:
out_proj = "+proj=longlat +datum=WGS84"
gcore.verbose(
"Assuming current projection as input, LL WGS84 as output ")
if ll_in:
out_proj = gcore.read_command("g.proj", flags="jf")
if proj_out:
if "+" in proj_out:
out_proj = proj_out
else:
gcore.fatal(_("Invalid PROJ.4 output specification"))
if not out_proj:
gcore.fatal(_("Missing output projection parameters "))
out_proj = out_proj.strip()
gcore.verbose("Output parameters: '%s'" % out_proj)
# set up input file
if coords:
x, y = coords.split(",")
tmpfile = gcore.tempfile()
fd = open(tmpfile, "w")
fd.write("%s%s%s\n" % (x, ifs, y))
fd.close()
inf = open(tmpfile)
else:
if input == "-":
infile = None
inf = sys.stdin
else:
infile = input
if not os.path.exists(infile):
gcore.fatal(_("Unable to read input data"))
inf = open(infile)
gcore.debug("input file=[%s]" % infile)
# set up output file
if not output:
outfile = None
outf = sys.stdout
else:
outfile = output
outf = open(outfile, "w")
gcore.debug("output file=[%s]" % outfile)
# set up output style
if not decimal:
outfmt = ["-w5"]
else:
outfmt = ["-f", "%.8f"]
if not copy_input:
copyinp = []
else:
copyinp = ["-E"]
# do the conversion
# Convert cs2cs DMS format to GRASS DMS format:
# cs2cs | sed -e 's/d/:/g' -e "s/'/:/g" -e 's/"//g'
cmd = ["cs2cs"
] + copyinp + outfmt + in_proj.split() + ["+to"] + out_proj.split()
p = gcore.Popen(cmd, stdin=gcore.PIPE, stdout=gcore.PIPE)
tr = TrThread(ifs, inf, p.stdin)
tr.start()
if not copy_input:
if include_header:
outf.write("x%sy%sz\n" % (ofs, ofs))
for line in p.stdout:
try:
xy, z = decode(line).split(" ", 1)
x, y = xy.split("\t")
except ValueError:
gcore.fatal(line)
outf.write("%s%s%s%s%s\n" %
(x.strip(), ofs, y.strip(), ofs, z.strip()))
else:
if include_header:
outf.write("input_x%sinput_y%sx%sy%sz\n" % (ofs, ofs, ofs, ofs))
for line in p.stdout:
inXYZ, x, rest = decode(line).split("\t")
inX, inY = inXYZ.split(" ")[:2]
y, z = rest.split(" ", 1)
outf.write("%s%s%s%s%s%s%s%s%s\n" % (
inX.strip(),
ofs,
inY.strip(),
ofs,
x.strip(),
ofs,
y.strip(),
ofs,
z.strip(),
))
p.wait()
if p.returncode != 0:
gcore.warning(
_("Projection transform probably failed, please investigate"))
def main():
vinput = options["input"]
columns = options["columns"].split(",")
binary = options["developed_column"]
level = options["subregions_column"]
sep = gutils.separator(options["separator"])
minim = int(options["min_variables"])
dredge = flags["d"]
if options["max_variables"]:
maxv = int(options["max_variables"])
else:
maxv = len(columns)
if dredge and minim > maxv:
gscript.fatal(
_("Minimum number of predictor variables is larger than maximum number"
))
global TMP_CSV, TMP_RSCRIPT, TMP_POT
TMP_CSV = gscript.tempfile(create=False) + ".csv"
TMP_RSCRIPT = gscript.tempfile()
include_level = True
distinct = gscript.read_command(
"v.db.select",
flags="c",
map=vinput,
columns="distinct {level}".format(level=level),
).strip()
if len(distinct.splitlines()) <= 1:
include_level = False
single_level = distinct.splitlines()[0]
with open(TMP_RSCRIPT, "w") as f:
f.write(rscript)
TMP_POT = gscript.tempfile(create=False) + "_potential.csv"
columns += [binary]
if include_level:
columns += [level]
where = "{c} IS NOT NULL".format(c=columns[0])
for c in columns[1:]:
where += " AND {c} IS NOT NULL".format(c=c)
gscript.run_command(
"v.db.select",
map=vinput,
columns=columns,
separator="comma",
where=where,
file=TMP_CSV,
)
if dredge:
gscript.info(_("Running automatic model selection ..."))
else:
gscript.info(_("Computing model..."))
cmd = [
"Rscript",
TMP_RSCRIPT,
"-i",
TMP_CSV,
"-r",
binary,
"-m",
str(minim),
"-x",
str(maxv),
"-o",
TMP_POT,
"-d",
"TRUE" if dredge else "FALSE",
]
if include_level:
cmd += ["-l", level]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
gscript.warning(gscript.decode(stderr))
if p.returncode != 0:
gscript.warning(gscript.decode(stderr))
gscript.fatal(_("Running R script failed, check messages above"))
gscript.info(_("Best model summary:"))
gscript.info("-------------------------")
gscript.message(gscript.decode(stdout))
with open(TMP_POT, "r") as fin, open(options["output"], "w") as fout:
i = 0
for line in fin.readlines():
row = line.strip().split("\t")
row = [each.strip('"') for each in row]
if i == 0:
row[0] = "ID"
row[1] = "Intercept"
if i == 1 and not include_level:
row[0] = single_level
fout.write(sep.join(row))
fout.write("\n")
i += 1
def main():
# Take into account those extra pixels we'll be a addin'
max_cols = int(options['maxcols']) - int(options['overlap'])
max_rows = int(options['maxrows']) - int(options['overlap'])
if max_cols == 0:
gcore.fatal(
_("It is not possible to set 'maxcols=%s' and "
"'overlap=%s'. Please set maxcols>overlap" %
(options['maxcols'], options['overlap'])))
elif max_rows == 0:
gcore.fatal(
_("It is not possible to set 'maxrows=%s' and "
"'overlap=%s'. Please set maxrows>overlap" %
(options['maxrows'], options['overlap'])))
# destination projection
if not options['destproj']:
dest_proj = gcore.read_command('g.proj', quiet=True,
flags='jf').rstrip('\n')
if not dest_proj:
gcore.fatal(_('g.proj failed'))
else:
dest_proj = options['destproj']
gcore.debug("Getting destination projection -> '%s'" % dest_proj)
# projection scale
if not options['destscale']:
ret = gcore.parse_command('g.proj', quiet=True, flags='j')
if not ret:
gcore.fatal(_('g.proj failed'))
if '+to_meter' in ret:
dest_scale = ret['+to_meter'].strip()
else:
gcore.warning(
_("Scale (%s) not found, assuming '1'") % '+to_meter')
dest_scale = '1'
else:
dest_scale = options['destscale']
gcore.debug('Getting destination projection scale -> %s' % dest_scale)
# set up the projections
srs_source = {
'proj': options['sourceproj'],
'scale': float(options['sourcescale'])
}
srs_dest = {'proj': dest_proj, 'scale': float(dest_scale)}
if options['region']:
gcore.run_command('g.region', quiet=True, region=options['region'])
dest_bbox = gcore.region()
gcore.debug('Getting destination region')
# output field separator
fs = separator(options['separator'])
# project the destination region into the source:
gcore.verbose('Projecting destination region into source...')
dest_bbox_points = bboxToPoints(dest_bbox)
dest_bbox_source_points, errors_dest = projectPoints(dest_bbox_points,
source=srs_dest,
dest=srs_source)
if len(dest_bbox_source_points) == 0:
gcore.fatal(
_("There are no tiles available. Probably the output "
"projection system it is not compatible with the "
"projection of the current location"))
source_bbox = pointsToBbox(dest_bbox_source_points)
gcore.verbose('Projecting source bounding box into destination...')
source_bbox_points = bboxToPoints(source_bbox)
source_bbox_dest_points, errors_source = projectPoints(source_bbox_points,
source=srs_source,
dest=srs_dest)
x_metric = 1 / dest_bbox['ewres']
y_metric = 1 / dest_bbox['nsres']
gcore.verbose('Computing length of sides of source bounding box...')
source_bbox_dest_lengths = sideLengths(source_bbox_dest_points, x_metric,
y_metric)
# Find the skewedness of the two directions.
# Define it to be greater than one
# In the direction (x or y) in which the world is least skewed (ie north south in lat long)
# Divide the world into strips. These strips are as big as possible contrained by max_
# In the other direction do the same thing.
# There's some recomputation of the size of the world that's got to come in
# here somewhere.
# For now, however, we are going to go ahead and request more data than is necessary.
# For small regions far from the critical areas of projections this makes very little difference
# in the amount of data gotten.
# We can make this efficient for big regions or regions near critical
# points later.
bigger = []
bigger.append(max(source_bbox_dest_lengths['x']))
bigger.append(max(source_bbox_dest_lengths['y']))
maxdim = (max_cols, max_rows)
# Compute the number and size of tiles to use in each direction
# I'm making fairly even sized tiles
# They differer from each other in height and width only by one cell
# I'm going to make the numbers all simpler and add this extra cell to
# every tile.
gcore.message(_('Computing tiling...'))
tiles = [-1, -1]
tile_base_size = [-1, -1]
tiles_extra_1 = [-1, -1]
tile_size = [-1, -1]
tileset_size = [-1, -1]
tile_size_overlap = [-1, -1]
for i in range(len(bigger)):
# make these into integers.
# round up
bigger[i] = int(bigger[i] + 1)
tiles[i] = int((bigger[i] / maxdim[i]) + 1)
tile_size[i] = tile_base_size[i] = int(bigger[i] / tiles[i])
tiles_extra_1[i] = int(bigger[i] % tiles[i])
# This is adding the extra pixel (remainder) to all of the tiles:
if tiles_extra_1[i] > 0:
tile_size[i] = tile_base_size[i] + 1
tileset_size[i] = int(tile_size[i] * tiles[i])
# Add overlap to tiles (doesn't effect tileset_size
tile_size_overlap[i] = tile_size[i] + int(options['overlap'])
gcore.verbose("There will be %d by %d tiles each %d by %d cells" %
(tiles[0], tiles[1], tile_size[0], tile_size[1]))
ximax = tiles[0]
yimax = tiles[1]
min_x = source_bbox['w']
min_y = source_bbox['s']
max_x = source_bbox['e']
max_y = source_bbox['n']
span_x = (max_x - min_x)
span_y = (max_y - min_y)
xi = 0
tile_bbox = {'w': -1, 's': -1, 'e': -1, 'n': -1}
if errors_dest > 0:
gcore.warning(
_("During computation %i tiles could not be created" %
errors_dest))
while xi < ximax:
tile_bbox['w'] = float(min_x) + (float(xi) * float(
tile_size[0]) / float(tileset_size[0])) * float(span_x)
tile_bbox['e'] = float(min_x) + (float(xi + 1) * float(
tile_size_overlap[0]) / float(tileset_size[0])) * float(span_x)
yi = 0
while yi < yimax:
tile_bbox['s'] = float(min_y) + (float(yi) * float(
tile_size[1]) / float(tileset_size[1])) * float(span_y)
tile_bbox['n'] = float(min_y) + (float(yi + 1) * float(
tile_size_overlap[1]) / float(tileset_size[1])) * float(span_y)
tile_bbox_points = bboxToPoints(tile_bbox)
tile_dest_bbox_points, errors = projectPoints(tile_bbox_points,
source=srs_source,
dest=srs_dest)
tile_dest_bbox = pointsToBbox(tile_dest_bbox_points)
if bboxesIntersect(tile_dest_bbox, dest_bbox):
if flags['w']:
print("bbox=%s,%s,%s,%s&width=%s&height=%s" %
(tile_bbox['w'], tile_bbox['s'], tile_bbox['e'],
tile_bbox['n'], tile_size_overlap[0],
tile_size_overlap[1]))
elif flags['g']:
print("w=%s;s=%s;e=%s;n=%s;cols=%s;rows=%s" %
(tile_bbox['w'], tile_bbox['s'], tile_bbox['e'],
tile_bbox['n'], tile_size_overlap[0],
tile_size_overlap[1]))
else:
print("%s%s%s%s%s%s%s%s%s%s%s" %
(tile_bbox['w'], fs, tile_bbox['s'], fs,
tile_bbox['e'], fs, tile_bbox['n'], fs,
tile_size_overlap[0], fs, tile_size_overlap[1]))
yi += 1
xi += 1
def main():
# Take into account those extra pixels we'll be a addin'
max_cols = int(options['maxcols']) - int(options['overlap'])
max_rows = int(options['maxrows']) - int(options['overlap'])
if max_cols == 0:
gcore.fatal(_("It is not possibile to set 'maxcols=%s' and "
"'overlap=%s'. Please set maxcols>overlap" %
(options['maxcols'], options['overlap'])))
elif max_rows == 0:
gcore.fatal(_("It is not possibile to set 'maxrows=%s' and "
"'overlap=%s'. Please set maxrows>overlap" %
(options['maxrows'], options['overlap'])))
# destination projection
if not options['destproj']:
dest_proj = gcore.read_command('g.proj',
quiet=True,
flags='jf').rstrip('\n')
if not dest_proj:
gcore.fatal(_('g.proj failed'))
else:
dest_proj = options['destproj']
gcore.debug("Getting destination projection -> '%s'" % dest_proj)
# projection scale
if not options['destscale']:
ret = gcore.parse_command('g.proj',
quiet=True,
flags='j')
if not ret:
gcore.fatal(_('g.proj failed'))
if '+to_meter' in ret:
dest_scale = ret['+to_meter'].strip()
else:
gcore.warning(
_("Scale (%s) not found, assuming '1'") %
'+to_meter')
dest_scale = '1'
else:
dest_scale = options['destscale']
gcore.debug('Getting destination projection scale -> %s' % dest_scale)
# set up the projections
srs_source = {'proj': options['sourceproj'],
'scale': float(options['sourcescale'])}
srs_dest = {'proj': dest_proj, 'scale': float(dest_scale)}
if options['region']:
gcore.run_command('g.region',
quiet=True,
region=options['region'])
dest_bbox = gcore.region()
gcore.debug('Getting destination region')
# output field separator
fs = separator(options['separator'])
# project the destination region into the source:
gcore.verbose('Projecting destination region into source...')
dest_bbox_points = bboxToPoints(dest_bbox)
dest_bbox_source_points, errors_dest = projectPoints(dest_bbox_points,
source=srs_dest,
dest=srs_source)
if len(dest_bbox_source_points) == 0:
gcore.fatal(_("There are no tiles available. Probably the output "
"projection system it is not compatible with the "
"projection of the current location"))
source_bbox = pointsToBbox(dest_bbox_source_points)
gcore.verbose('Projecting source bounding box into destination...')
source_bbox_points = bboxToPoints(source_bbox)
source_bbox_dest_points, errors_source = projectPoints(source_bbox_points,
source=srs_source,
dest=srs_dest)
x_metric = 1 / dest_bbox['ewres']
y_metric = 1 / dest_bbox['nsres']
gcore.verbose('Computing length of sides of source bounding box...')
source_bbox_dest_lengths = sideLengths(source_bbox_dest_points,
x_metric, y_metric)
# Find the skewedness of the two directions.
# Define it to be greater than one
# In the direction (x or y) in which the world is least skewed (ie north south in lat long)
# Divide the world into strips. These strips are as big as possible contrained by max_
# In the other direction do the same thing.
# Theres some recomputation of the size of the world that's got to come in
# here somewhere.
# For now, however, we are going to go ahead and request more data than is necessary.
# For small regions far from the critical areas of projections this makes very little difference
# in the amount of data gotten.
# We can make this efficient for big regions or regions near critical
# points later.
bigger = []
bigger.append(max(source_bbox_dest_lengths['x']))
bigger.append(max(source_bbox_dest_lengths['y']))
maxdim = (max_cols, max_rows)
# Compute the number and size of tiles to use in each direction
# I'm making fairly even sized tiles
# They differer from each other in height and width only by one cell
# I'm going to make the numbers all simpler and add this extra cell to
# every tile.
gcore.message(_('Computing tiling...'))
tiles = [-1, -1]
tile_base_size = [-1, -1]
tiles_extra_1 = [-1, -1]
tile_size = [-1, -1]
tileset_size = [-1, -1]
tile_size_overlap = [-1, -1]
for i in range(len(bigger)):
# make these into integers.
# round up
bigger[i] = int(bigger[i] + 1)
tiles[i] = int((bigger[i] / maxdim[i]) + 1)
tile_size[i] = tile_base_size[i] = int(bigger[i] / tiles[i])
tiles_extra_1[i] = int(bigger[i] % tiles[i])
# This is adding the extra pixel (remainder) to all of the tiles:
if tiles_extra_1[i] > 0:
tile_size[i] = tile_base_size[i] + 1
tileset_size[i] = int(tile_size[i] * tiles[i])
# Add overlap to tiles (doesn't effect tileset_size
tile_size_overlap[i] = tile_size[i] + int(options['overlap'])
gcore.verbose("There will be %d by %d tiles each %d by %d cells" %
(tiles[0], tiles[1], tile_size[0], tile_size[1]))
ximax = tiles[0]
yimax = tiles[1]
min_x = source_bbox['w']
min_y = source_bbox['s']
max_x = source_bbox['e']
max_y = source_bbox['n']
span_x = (max_x - min_x)
span_y = (max_y - min_y)
xi = 0
tile_bbox = {'w': -1, 's': -1, 'e': -1, 'n': -1}
if errors_dest > 0:
gcore.warning(_("During computation %i tiles could not be created" %
errors_dest))
while xi < ximax:
tile_bbox['w'] = float(
min_x) + (float(xi) * float(tile_size[0]) / float(tileset_size[0])) * float(span_x)
tile_bbox['e'] = float(min_x) + (float(xi + 1) * float(tile_size_overlap[0]
) / float(tileset_size[0])) * float(span_x)
yi = 0
while yi < yimax:
tile_bbox['s'] = float(
min_y) + (float(yi) * float(tile_size[1]) / float(tileset_size[1])) * float(span_y)
tile_bbox['n'] = float(min_y) + (
float(yi + 1) * float(tile_size_overlap[1]) /
float(tileset_size[1])) * float(span_y)
tile_bbox_points = bboxToPoints(tile_bbox)
tile_dest_bbox_points, errors = projectPoints(tile_bbox_points,
source=srs_source,
dest=srs_dest)
tile_dest_bbox = pointsToBbox(tile_dest_bbox_points)
if bboxesIntersect(tile_dest_bbox, dest_bbox):
if flags['w']:
print("bbox=%s,%s,%s,%s&width=%s&height=%s" %
(tile_bbox['w'], tile_bbox['s'], tile_bbox['e'],
tile_bbox['n'], tile_size_overlap[0],
tile_size_overlap[1]))
elif flags['g']:
print("w=%s;s=%s;e=%s;n=%s;cols=%s;rows=%s" %
(tile_bbox['w'], tile_bbox['s'], tile_bbox['e'],
tile_bbox['n'], tile_size_overlap[0],
tile_size_overlap[1]))
else:
print("%s%s%s%s%s%s%s%s%s%s%s" %
(tile_bbox['w'], fs, tile_bbox['s'], fs,
tile_bbox['e'], fs, tile_bbox['n'], fs,
tile_size_overlap[0], fs, tile_size_overlap[1]))
yi += 1
xi += 1
def main():
global tmp
fs = separator(options["separator"])
threeD = flags["z"]
if threeD:
do3D = "z"
else:
do3D = ""
tmp = grass.tempfile()
# set up input file
if options["input"] == "-":
infile = None
inf = sys.stdin
else:
infile = options["input"]
if not os.path.exists(infile):
grass.fatal(_("Unable to read input file <%s>") % infile)
grass.debug("input file=[%s]" % infile)
if not infile:
# read from stdin and write to tmpfile (v.in.mapgen wants a real file)
outf = open(tmp, "w")
for line in inf:
if len(line.lstrip()) == 0 or line[0] == "#":
continue
outf.write(line.replace(fs, " "))
outf.close()
runfile = tmp
else:
# read from a real file
if fs == " ":
runfile = infile
else:
inf = open(infile)
outf = open(tmp, "w")
for line in inf:
if len(line.lstrip()) == 0 or line[0] == "#":
continue
outf.write(line.replace(fs, " "))
inf.close()
outf.close()
runfile = tmp
# check that there are at least two columns (three if -z is given)
inf = open(runfile)
for line in inf:
if len(line.lstrip()) == 0 or line[0] == "#":
continue
numcols = len(line.split())
break
inf.close()
if (do3D and numcols < 3) or (not do3D and numcols < 2):
grass.fatal(_("Not enough data columns. (incorrect fs setting?)"))
grass.run_command(
"v.in.mapgen", flags="f" + do3D, input=runfile, output=options["output"]
)
def main():
# if no output filename, output to stdout
input = options['input']
player = int(options['player'])
output = options['output']
idcolumn = options['idcolumn'] if options['idcolumn'] else False
sep = separator(options['separator'])
bidirectional = flags['b']
global tempmapname
tempmapname = 'neighborhoodmatrix_tempmap_%d' % os.getpid()
#TODO: automatically determine the first available layer in file
blayer = player + 1
gscript.run_command('v.category',
input=input,
output=tempmapname,
option='add',
layer=blayer,
type='boundary',
quiet=True,
overwrite=True)
vtodb_results = gscript.read_command('v.to.db',
flags='p',
map=tempmapname,
type='boundary',
option='sides',
layer=blayer,
qlayer=player,
quiet=True)
#put result into a list of integer pairs
temp_neighbors = []
for line in vtodb_results.splitlines():
if line.split('|')[1] != '-1' and line.split('|')[2] != '-1':
temp_neighbors.append(
[int(line.split('|')[1]),
int(line.split('|')[2])])
#temp_neighbors.sort()
#if user wants bidirectional matrix, add the inversed pairs to the original
if bidirectional:
neighbors_reversed = []
for pair in temp_neighbors:
neighbors_reversed.append([pair[1], pair[0]])
temp_neighbors += neighbors_reversed
#uniqify the list of integer pairs
neighbors = sorted(
[list(x) for x in set(tuple(x) for x in temp_neighbors)])
currentcat = ''
if output and output != '-':
out = open(output, 'w')
for pair in neighbors:
if idcolumn:
# While pair[0] stays the same we don't have to call v.db.select
# again and again to get the id
if currentcat != pair[0]:
currentcat = pair[0]
fromid = gscript.read_command('v.db.select',
map=input,
column=idcolumn,
where="cat=%d" % pair[0],
layer=player,
flags="c",
quiet=True).rstrip()
toid = gscript.read_command('v.db.select',
map=input,
column=idcolumn,
where="cat=%d" % pair[1],
layer=player,
flags="c",
quiet=True).rstrip()
if output and output != '-':
out.write(fromid + sep + toid + '\n')
else:
print((fromid + sep + toid))
else:
if output and output != '-':
out.write(str(pair[0]) + sep + str(pair[1]) + '\n')
else:
print((str(pair[0]) + sep + str(pair[1])))
if output and output != '-':
out.close()
sys.exit()
def main():
# if no output filename, output to stdout
input = options["input"]
player = int(options["player"])
output = options["output"]
idcolumn = options["idcolumn"] if options["idcolumn"] else False
sep = separator(options["separator"])
bidirectional = flags["b"]
global tempmapname
tempmapname = "neighborhoodmatrix_tempmap_%d" % os.getpid()
# TODO: automatically determine the first available layer in file
blayer = player + 1
gscript.run_command(
"v.category",
input=input,
output=tempmapname,
option="add",
layer=blayer,
type="boundary",
quiet=True,
overwrite=True,
)
vtodb_results = gscript.read_command(
"v.to.db",
flags="p",
map=tempmapname,
type="boundary",
option="sides",
layer=blayer,
qlayer=player,
quiet=True,
)
# put result into a list of integer pairs
temp_neighbors = []
for line in vtodb_results.splitlines():
if line.split("|")[1] != "-1" and line.split("|")[2] != "-1":
temp_neighbors.append(
[int(line.split("|")[1]),
int(line.split("|")[2])])
# temp_neighbors.sort()
# if user wants bidirectional matrix, add the inversed pairs to the original
if bidirectional:
neighbors_reversed = []
for pair in temp_neighbors:
neighbors_reversed.append([pair[1], pair[0]])
temp_neighbors += neighbors_reversed
# uniqify the list of integer pairs
neighbors = sorted(
[list(x) for x in set(tuple(x) for x in temp_neighbors)])
currentcat = ""
if output and output != "-":
out = open(output, "w")
for pair in neighbors:
if idcolumn:
# While pair[0] stays the same we don't have to call v.db.select
# again and again to get the id
if currentcat != pair[0]:
currentcat = pair[0]
fromid = gscript.read_command(
"v.db.select",
map=input,
column=idcolumn,
where="cat=%d" % pair[0],
layer=player,
flags="c",
quiet=True,
).rstrip()
toid = gscript.read_command(
"v.db.select",
map=input,
column=idcolumn,
where="cat=%d" % pair[1],
layer=player,
flags="c",
quiet=True,
).rstrip()
if output and output != "-":
out.write(fromid + sep + toid + "\n")
else:
print((fromid + sep + toid))
else:
if output and output != "-":
out.write(str(pair[0]) + sep + str(pair[1]) + "\n")
else:
print((str(pair[0]) + sep + str(pair[1])))
if output and output != "-":
out.close()
sys.exit()
def main():
check_addon_installed('r.object.geometry', fatal=True)
dev_start = options['development_start']
dev_end = options['development_end']
only_file = flags['l']
patches_per_subregion = flags['s']
if not only_file:
repeat = int(options['repeat'])
compactness_means = [float(each) for each in options['compactness_mean'].split(',')]
compactness_ranges = [float(each) for each in options['compactness_range'].split(',')]
discount_factors = [float(each) for each in options['discount_factor'].split(',')]
patches_file = options['patch_sizes']
threshold = float(options['patch_threshold'])
sep = gutils.separator(options['separator'])
# v.clean removes size <= threshold, we want to keep size == threshold
threshold -= 1e-6
# compute cell size
region = gcore.region()
res = (region['nsres'] + region['ewres']) / 2.
coeff = float(gcore.parse_command('g.proj', flags='g')['meters'])
cell_size = res * res * coeff * coeff
tmp_name = 'tmp_futures_calib_' + str(os.getpid()) + '_'
global TMP
orig_patch_diff = tmp_name + 'orig_patch_diff'
TMP.append(orig_patch_diff)
tmp_clump = tmp_name + 'tmp_clump'
TMP.append(tmp_clump)
if patches_per_subregion:
tmp_cat_clump = tmp_name + 'tmp_cat_clump'
TMP.append(tmp_cat_clump)
gcore.message(_("Analyzing original patches..."))
diff_development(dev_start, dev_end, options['subregions'], orig_patch_diff)
data = write_data = patch_analysis(orig_patch_diff, threshold, tmp_clump)
if patches_per_subregion:
subregions_data = patch_analysis_per_subregion(orig_patch_diff, options['subregions'],
threshold, tmp_clump, tmp_cat_clump)
# if there is just one column, write the previous analysis result
if len(subregions_data.keys()) > 1:
write_data = subregions_data
write_patches_file(write_data, cell_size, patches_file, sep)
if only_file:
return
area, perimeter = data.T
compact = compactness(area, perimeter)
# area histogram
area = area / cell_size
bin_width = 1. # automatic ways to determine bin width do not perform well in this case
hist_bins_area_orig = int(np.ptp(area) / bin_width)
hist_range_area_orig = (np.min(area), np.max(area))
histogram_area_orig, _edges = np.histogram(area, bins=hist_bins_area_orig,
range=hist_range_area_orig, density=True)
histogram_area_orig = histogram_area_orig * 100 # to get percentage for readability
# compactness histogram
bin_width = 0.1
hist_bins_compactness_orig = int(np.ptp(compact) / bin_width)
hist_range_compactness_orig = (np.min(compact), np.max(compact))
histogram_compactness_orig, _edges = np.histogram(compact, bins=hist_bins_compactness_orig,
range=hist_range_compactness_orig, density=True)
histogram_compactness_orig = histogram_compactness_orig * 100 # to get percentage for readability
seed = int(options['random_seed'])
nprocs = int(options['nprocs'])
count = 0
proc_count = 0
queue_list = []
proc_list = []
num_all = len(compactness_means) * len(compactness_ranges) * len(discount_factors)
with open(options['calibration_results'], 'w') as f:
for com_mean in compactness_means:
for com_range in compactness_ranges:
for discount_factor in discount_factors:
count += 1
q = Queue()
p = Process(target=run_one_combination,
args=(count, num_all, repeat, seed, dev_start, com_mean, com_range,
discount_factor, patches_file, options, threshold,
hist_bins_area_orig, hist_range_area_orig, hist_bins_compactness_orig,
hist_range_compactness_orig, cell_size, histogram_area_orig, histogram_compactness_orig,
tmp_name, q))
p.start()
queue_list.append(q)
proc_list.append(p)
proc_count += 1
seed += 1
if proc_count == nprocs or count == num_all:
for i in range(proc_count):
proc_list[i].join()
data = queue_list[i].get()
if not data:
continue
f.write(','.join([str(data['input_discount_factor']), str(data['area_distance']),
str(data['input_compactness_mean']), str(data['input_compactness_range']),
str(data['compactness_distance'])]))
f.write('\n')
f.flush()
proc_count = 0
proc_list = []
queue_list = []
# compute combined normalized error
process_calibration(options['calibration_results'])
def main():
coords = options['coordinates']
input = options['input']
output = options['output']
fs = options['separator']
proj_in = options['proj_in']
proj_out = options['proj_out']
ll_in = flags['i']
ll_out = flags['o']
decimal = flags['d']
copy_input = flags['e']
include_header = flags['c']
#### check for cs2cs
if not grass.find_program('cs2cs'):
grass.fatal(_("cs2cs program not found, install PROJ.4 first: http://proj.maptools.org"))
#### check for overenthusiasm
if proj_in and ll_in:
grass.fatal(_("Choose only one input parameter method"))
if proj_out and ll_out:
grass.fatal(_("Choose only one output parameter method"))
if ll_in and ll_out:
grass.fatal(_("Choise only one auto-projection parameter method"))
if output and not grass.overwrite() and os.path.exists(output):
grass.fatal(_("Output file already exists"))
if not coords and not input:
grass.fatal(_("One of <coordinates> and <input> must be given"))
if coords and input:
grass.fatal(_("Options <coordinates> and <input> are mutually exclusive"))
#### parse field separator
# FIXME: input_x,y needs to split on multiple whitespace between them
if fs == ',':
ifs = ofs = ','
else:
try:
ifs, ofs = fs.split(',')
except ValueError:
ifs = ofs = fs
ifs = separator(ifs)
ofs = separator(ofs)
#### set up projection params
s = grass.read_command("g.proj", flags='j')
kv = parse_key_val(s)
if "XY location" in kv['+proj'] and (ll_in or ll_out):
grass.fatal(_("Unable to project to or from a XY location"))
in_proj = None
if ll_in:
in_proj = "+proj=longlat +datum=WGS84"
grass.verbose("Assuming LL WGS84 as input, current projection as output ")
if ll_out:
in_proj = grass.read_command('g.proj', flags = 'jf')
if proj_in:
in_proj = proj_in
if not in_proj:
grass.verbose("Assuming current location as input")
in_proj = grass.read_command('g.proj', flags = 'jf')
in_proj = in_proj.strip()
grass.verbose("Input parameters: '%s'" % in_proj)
out_proj = None
if ll_out:
out_proj = "+proj=longlat +datum=WGS84"
grass.verbose("Assuming current projection as input, LL WGS84 as output ")
if ll_in:
out_proj = grass.read_command('g.proj', flags = 'jf')
if proj_out:
out_proj = proj_out
if not out_proj:
grass.fatal(_("Missing output projection parameters "))
out_proj = out_proj.strip()
grass.verbose("Output parameters: '%s'" % out_proj)
#### set up input file
if coords:
x, y = coords.split(',')
tmpfile = grass.tempfile()
fd = open(tmpfile, "w")
fd.write("%s%s%s\n" % (x, ifs, y))
fd.close()
inf = file(tmpfile)
else:
if input == '-':
infile = None
inf = sys.stdin
else:
infile = input
if not os.path.exists(infile):
grass.fatal(_("Unable to read input data"))
inf = file(infile)
grass.debug("input file=[%s]" % infile)
#### set up output file
if not output:
outfile = None
outf = sys.stdout
else:
outfile = output
outf = open(outfile, 'w')
grass.debug("output file=[%s]" % outfile)
#### set up output style
if not decimal:
outfmt = ["-w5"]
else:
outfmt = ["-f", "%.8f"]
if not copy_input:
copyinp = []
else:
copyinp = ["-E"]
#### do the conversion
# Convert cs2cs DMS format to GRASS DMS format:
# cs2cs | sed -e 's/d/:/g' -e "s/'/:/g" -e 's/"//g'
cmd = ['cs2cs'] + copyinp + outfmt + in_proj.split() + ['+to'] + out_proj.split()
p = grass.Popen(cmd, stdin = grass.PIPE, stdout = grass.PIPE)
tr = TrThread(ifs, inf, p.stdin)
tr.start()
if not copy_input:
if include_header:
outf.write("x%sy%sz\n" % (ofs, ofs))
for line in p.stdout:
try:
xy, z = line.split(' ', 1)
x, y = xy.split('\t')
except ValueError:
grass.fatal(line)
outf.write('%s%s%s%s%s\n' % \
(x.strip(), ofs, y.strip(), ofs, z.strip()))
else:
if include_header:
outf.write("input_x%sinput_y%sx%sy%sz\n" % (ofs, ofs, ofs, ofs))
for line in p.stdout:
inXYZ, x, rest = line.split('\t')
inX, inY = inXYZ.split(' ')[:2]
y, z = rest.split(' ', 1)
outf.write('%s%s%s%s%s%s%s%s%s\n' % \
(inX.strip(), ofs, inY.strip(), ofs, x.strip(), \
ofs, y.strip(), ofs, z.strip()))
p.wait()
if p.returncode != 0:
grass.warning(_("Projection transform probably failed, please investigate"))
def main():
coords = options['coordinates']
input = options['input']
output = options['output']
fs = options['separator']
proj_in = options['proj_in']
proj_out = options['proj_out']
ll_in = flags['i']
ll_out = flags['o']
decimal = flags['d']
copy_input = flags['e']
include_header = flags['c']
# check for cs2cs
if not gcore.find_program('cs2cs'):
gcore.fatal(
_("cs2cs program not found, install PROJ.4 first: \
http://proj.maptools.org"))
# check for overenthusiasm
if proj_in and ll_in:
gcore.fatal(_("Choose only one input parameter method"))
if proj_out and ll_out:
gcore.fatal(_("Choose only one output parameter method"))
if ll_in and ll_out:
gcore.fatal(_("Choose only one auto-projection parameter method"))
if output and not gcore.overwrite() and os.path.exists(output):
gcore.fatal(_("Output file already exists"))
if not coords and not input:
gcore.fatal(_("One of <coordinates> and <input> must be given"))
if coords and input:
gcore.fatal(
_("Options <coordinates> and <input> are mutually exclusive"))
# parse field separator
# FIXME: input_x,y needs to split on multiple whitespace between them
if fs == ',':
ifs = ofs = ','
else:
try:
ifs, ofs = fs.split(',')
except ValueError:
ifs = ofs = fs
ifs = separator(ifs)
ofs = separator(ofs)
# set up projection params
s = gcore.read_command("g.proj", flags='j')
kv = parse_key_val(s)
if "XY location" in kv['+proj'] and (ll_in or ll_out):
gcore.fatal(_("Unable to project to or from a XY location"))
in_proj = None
if ll_in:
in_proj = "+proj=longlat +datum=WGS84"
gcore.verbose(
"Assuming LL WGS84 as input, current projection as output ")
if ll_out:
in_proj = gcore.read_command('g.proj', flags='jf')
if proj_in:
if '+' in proj_in:
in_proj = proj_in
else:
gcore.fatal(_("Invalid PROJ.4 input specification"))
if not in_proj:
gcore.verbose("Assuming current location as input")
in_proj = gcore.read_command('g.proj', flags='jf')
in_proj = in_proj.strip()
gcore.verbose("Input parameters: '%s'" % in_proj)
out_proj = None
if ll_out:
out_proj = "+proj=longlat +datum=WGS84"
gcore.verbose(
"Assuming current projection as input, LL WGS84 as output ")
if ll_in:
out_proj = gcore.read_command('g.proj', flags='jf')
if proj_out:
if '+' in proj_out:
out_proj = proj_out
else:
gcore.fatal(_("Invalid PROJ.4 output specification"))
if not out_proj:
gcore.fatal(_("Missing output projection parameters "))
out_proj = out_proj.strip()
gcore.verbose("Output parameters: '%s'" % out_proj)
# set up input file
if coords:
x, y = coords.split(',')
tmpfile = gcore.tempfile()
fd = open(tmpfile, "w")
fd.write("%s%s%s\n" % (x, ifs, y))
fd.close()
inf = file(tmpfile)
else:
if input == '-':
infile = None
inf = sys.stdin
else:
infile = input
if not os.path.exists(infile):
gcore.fatal(_("Unable to read input data"))
inf = file(infile)
gcore.debug("input file=[%s]" % infile)
# set up output file
if not output:
outfile = None
outf = sys.stdout
else:
outfile = output
outf = open(outfile, 'w')
gcore.debug("output file=[%s]" % outfile)
# set up output style
if not decimal:
outfmt = ["-w5"]
else:
outfmt = ["-f", "%.8f"]
if not copy_input:
copyinp = []
else:
copyinp = ["-E"]
# do the conversion
# Convert cs2cs DMS format to GRASS DMS format:
# cs2cs | sed -e 's/d/:/g' -e "s/'/:/g" -e 's/"//g'
cmd = ['cs2cs'] + copyinp + outfmt + \
in_proj.split() + ['+to'] + out_proj.split()
p = gcore.Popen(cmd, stdin=gcore.PIPE, stdout=gcore.PIPE)
tr = TrThread(ifs, inf, p.stdin)
tr.start()
if not copy_input:
if include_header:
outf.write("x%sy%sz\n" % (ofs, ofs))
for line in p.stdout:
try:
xy, z = line.split(' ', 1)
x, y = xy.split('\t')
except ValueError:
gcore.fatal(line)
outf.write('%s%s%s%s%s\n' %
(x.strip(), ofs, y.strip(), ofs, z.strip()))
else:
if include_header:
outf.write("input_x%sinput_y%sx%sy%sz\n" % (ofs, ofs, ofs, ofs))
for line in p.stdout:
inXYZ, x, rest = line.split('\t')
inX, inY = inXYZ.split(' ')[:2]
y, z = rest.split(' ', 1)
outf.write('%s%s%s%s%s%s%s%s%s\n' %
(inX.strip(), ofs, inY.strip(), ofs, x.strip(), ofs,
y.strip(), ofs, z.strip()))
p.wait()
if p.returncode != 0:
gcore.warning(
_("Projection transform probably failed, please investigate"))
def main():
global tmp
fs = separator(options['separator'])
threeD = flags['z']
prog = 'v.in.lines'
if threeD:
do3D = 'z'
else:
do3D = ''
tmp = grass.tempfile()
# set up input file
if options['input'] == '-':
infile = None
inf = sys.stdin
else:
infile = options['input']
if not os.path.exists(infile):
grass.fatal(_("Unable to read input file <%s>") % infile)
grass.debug("input file=[%s]" % infile)
if not infile:
# read from stdin and write to tmpfile (v.in.mapgen wants a real file)
outf = file(tmp, 'w')
for line in inf:
if len(line.lstrip()) == 0 or line[0] == '#':
continue
outf.write(line.replace(fs, ' '))
outf.close()
runfile = tmp
else:
# read from a real file
if fs == ' ':
runfile = infile
else:
inf = file(infile)
outf = file(tmp, 'w')
for line in inf:
if len(line.lstrip()) == 0 or line[0] == '#':
continue
outf.write(line.replace(fs, ' '))
inf.close()
outf.close()
runfile = tmp
# check that there are at least two columns (three if -z is given)
inf = file(runfile)
for line in inf:
if len(line.lstrip()) == 0 or line[0] == '#':
continue
numcols = len(line.split())
break
inf.close()
if (do3D and numcols < 3) or (not do3D and numcols < 2):
grass.fatal(_("Not enough data columns. (incorrect fs setting?)"))
grass.run_command('v.in.mapgen',
flags='f' + do3D,
input=runfile,
output=options['output'])
def main():
vinput = options["input"]
columns = options["columns"].split(",")
binary = options["developed_column"]
level = options["subregions_column"]
random = options["random_column"]
sep = gutils.separator(options["separator"])
minim = int(options["min_variables"])
dredge = flags["d"]
nprocs = int(options["nprocs"])
fixed_columns = (options["fixed_columns"].split(",")
if options["fixed_columns"] else [])
for each in fixed_columns:
if each not in columns:
gscript.fatal(
_("Fixed predictor {} not among predictors specified in option 'columns'"
).format(each))
if options["max_variables"]:
maxv = int(options["max_variables"])
else:
maxv = len(columns)
if dredge and minim > maxv:
gscript.fatal(
_("Minimum number of predictor variables is larger than maximum number"
))
if not gscript.find_program("Rscript", "--version"):
gscript.fatal(
_("Rscript required for running r.futures.potential, but not found. "
"Make sure you have R installed and added to the PATH."))
global TMP_CSV, TMP_RSCRIPT, TMP_POT, TMP_DREDGE
TMP_CSV = gscript.tempfile(create=False) + ".csv"
TMP_RSCRIPT = gscript.tempfile()
include_level = True
distinct = gscript.read_command(
"v.db.select",
flags="c",
map=vinput,
columns="distinct {level}".format(level=level),
).strip()
if len(distinct.splitlines()) <= 1:
include_level = False
single_level = distinct.splitlines()[0]
with open(TMP_RSCRIPT, "w") as f:
f.write(rscript)
TMP_POT = gscript.tempfile(create=False) + "_potential.csv"
TMP_DREDGE = gscript.tempfile(create=False) + "_dredge.csv"
columns += [binary]
if include_level:
columns += [level]
if random:
columns += [random]
# filter duplicates
columns = list(dict.fromkeys(columns))
where = "{c} IS NOT NULL".format(c=columns[0])
for c in columns[1:]:
where += " AND {c} IS NOT NULL".format(c=c)
gscript.run_command(
"v.db.select",
map=vinput,
columns=columns,
separator="comma",
where=where,
file=TMP_CSV,
)
if dredge:
gscript.info(_("Running automatic model selection ..."))
else:
gscript.info(_("Computing model..."))
cmd = [
"Rscript",
TMP_RSCRIPT,
"-i",
TMP_CSV,
"-r",
binary,
"-o",
TMP_POT,
"-p",
",".join(columns),
"-m",
str(minim),
"-x",
str(maxv),
"-d",
"TRUE" if dredge else "FALSE",
"-n",
str(nprocs),
]
if include_level:
cmd += ["-l", level]
if random:
cmd += ["-a", random]
if dredge and fixed_columns:
cmd += ["-f", ",".join(fixed_columns)]
if dredge and options["dredge_output"]:
cmd += ["-e", TMP_DREDGE]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
if stderr:
gscript.warning(gscript.decode(stderr))
if p.returncode != 0:
gscript.fatal(_("Running R script failed, check messages above"))
gscript.info(_("Best model summary:"))
gscript.info("-------------------------")
gscript.message(gscript.decode(stdout))
# note: this would be better with pandas, but adds dependency
with open(TMP_POT, "r") as fin, open(options["output"], "w") as fout:
i = 0
for line in fin.readlines():
row = line.strip().split("\t")
row = [each.strip('"') for each in row]
if i == 0:
row[0] = "ID"
if include_level and random:
row[2] = row[1]
row[1] = "Intercept"
if i == 1 and not include_level:
row[0] = single_level
if i >= 1:
if include_level:
# devpressure needs to be after intercept
if random:
row[2], row[1] = row[1], row[2]
else:
row[0] = single_level
fout.write(sep.join(row))
fout.write("\n")
i += 1
if options["dredge_output"]:
with open(TMP_DREDGE, "r") as fin, open(options["dredge_output"],
"w") as fout:
i = 0
for line in fin.readlines():
row = line.strip().split(",")
row = [each.strip('"') for each in row]
if i == 0:
row[0] = "ID"
row[1] = "Intercept"
fout.write(sep.join(row))
fout.write("\n")
i += 1
def main():
developments = options['development'].split(',')
observed_popul_file = options['observed_population']
projected_popul_file = options['projected_population']
sep = gutils.separator(options['separator'])
subregions = options['subregions']
methods = options['method'].split(',')
plot = options['plot']
simulation_times = [
float(each) for each in options['simulation_times'].split(',')
]
for each in methods:
if each in ('exp_approach', 'logarithmic2'):
try:
from scipy.optimize import curve_fit
except ImportError:
gcore.fatal(
_("Importing scipy failed. Method '{m}' is not available").
format(m=each))
# exp approach needs at least 3 data points
if len(developments) <= 2 and ('exp_approach' in methods
or 'logarithmic2' in methods):
gcore.fatal(_("Not enough data for method 'exp_approach'"))
if len(developments) == 3 and ('exp_approach' in methods
and 'logarithmic2' in methods):
gcore.warning(
_("Can't decide between 'exp_approach' and 'logarithmic2' methods"
" because both methods can have exact solutions for 3 data points resulting in RMSE = 0"
))
observed_popul = np.genfromtxt(observed_popul_file,
dtype=float,
delimiter=sep,
names=True)
projected_popul = np.genfromtxt(projected_popul_file,
dtype=float,
delimiter=sep,
names=True)
year_col = observed_popul.dtype.names[0]
observed_times = observed_popul[year_col]
year_col = projected_popul.dtype.names[0]
projected_times = projected_popul[year_col]
if len(developments) != len(observed_times):
gcore.fatal(
_("Number of development raster maps doesn't not correspond to the number of observed times"
))
# gather developed cells in subregions
gcore.info(_("Computing number of developed cells..."))
table_developed = {}
subregionIds = set()
for i in range(len(observed_times)):
gcore.percent(i, len(observed_times), 1)
data = gcore.read_command('r.univar',
flags='gt',
zones=subregions,
map=developments[i])
for line in data.splitlines():
stats = line.split('|')
if stats[0] == 'zone':
continue
subregionId, developed_cells = stats[0], int(stats[12])
subregionIds.add(subregionId)
if i == 0:
table_developed[subregionId] = []
table_developed[subregionId].append(developed_cells)
gcore.percent(1, 1, 1)
subregionIds = sorted(list(subregionIds))
# linear interpolation between population points
population_for_simulated_times = {}
for subregionId in table_developed.keys():
population_for_simulated_times[subregionId] = np.interp(
x=simulation_times,
xp=np.append(observed_times, projected_times),
fp=np.append(observed_popul[subregionId],
projected_popul[subregionId]))
# regression
demand = {}
i = 0
if plot:
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
n_plots = np.ceil(np.sqrt(len(subregionIds)))
fig = plt.figure(figsize=(5 * n_plots, 5 * n_plots))
for subregionId in subregionIds:
i += 1
rmse = dict()
predicted = dict()
simulated = dict()
coeff = dict()
for method in methods:
# observed population points for subregion
reg_pop = observed_popul[subregionId]
simulated[method] = np.array(
population_for_simulated_times[subregionId])
if method in ('exp_approach', 'logarithmic2'):
# we have to scale it first
y = np.array(table_developed[subregionId])
magn = float(
np.power(
10,
max(magnitude(np.max(reg_pop)), magnitude(np.max(y)))))
x = reg_pop / magn
y = y / magn
if method == 'exp_approach':
initial = (
0.5, np.mean(x), np.mean(y)
) # this seems to work best for our data for exp_approach
elif method == 'logarithmic2':
popt, pcov = curve_fit(logarithmic, x, y)
initial = (popt[0], popt[1], 0)
with np.errstate(
invalid='warn'
): # when 'raise' it stops every time on FloatingPointError
try:
popt, pcov = curve_fit(globals()[method],
x,
y,
p0=initial)
if np.isnan(popt).any():
raise RuntimeError
# would result in nans in predicted
if method == 'logarithmic2' and np.any(
simulated[method] / magn <= popt[-1]):
raise RuntimeError
except (FloatingPointError, RuntimeError):
rmse[
method] = sys.maxsize # so that other method is selected
gcore.warning(
_("Method '{m}' cannot converge for subregion {reg}"
.format(m=method, reg=subregionId)))
if len(methods) == 1:
gcore.fatal(
_("Method '{m}' failed for subregion {reg},"
" please select at least one other method").
format(m=method, reg=subregionId))
else:
predicted[method] = globals()[method](
simulated[method] / magn, *popt) * magn
r = globals()[method](
x, *popt) * magn - table_developed[subregionId]
coeff[method] = popt
if len(reg_pop) > 3:
rmse[method] = np.sqrt(
(np.sum(r * r) / (len(reg_pop) - 3)))
else:
rmse[method] = 0
else:
if method == 'logarithmic':
reg_pop = np.log(reg_pop)
if method == 'exponential':
y = np.log(table_developed[subregionId])
else:
y = table_developed[subregionId]
A = np.vstack((reg_pop, np.ones(len(reg_pop)))).T
npversion = [int(x) for x in np.__version__.split('.')]
if npversion >= [1, 14, 0]:
rcond = None
else:
rcond = -1
m, c = np.linalg.lstsq(A, y, rcond=rcond)[0] # y = mx + c
coeff[method] = m, c
if method == 'logarithmic':
with np.errstate(invalid='ignore', divide='ignore'):
predicted[method] = np.where(
simulated[method] > 1,
np.log(simulated[method]) * m + c, 0)
predicted[method] = np.where(predicted[method] > 0,
predicted[method], 0)
r = (reg_pop * m + c) - table_developed[subregionId]
elif method == 'exponential':
predicted[method] = np.exp(m * simulated[method] + c)
r = np.exp(m * reg_pop + c) - table_developed[subregionId]
else: # linear
predicted[method] = simulated[method] * m + c
r = (reg_pop * m + c) - table_developed[subregionId]
# RMSE
if len(reg_pop) > 2:
rmse[method] = np.sqrt(
(np.sum(r * r) / (len(reg_pop) - 2)))
else:
rmse[method] = 0
method = min(rmse, key=rmse.get)
gcore.verbose(
_("Method '{meth}' was selected for subregion {reg}").format(
meth=method, reg=subregionId))
# write demand
demand[subregionId] = predicted[method]
demand[subregionId] = np.diff(demand[subregionId])
if np.any(demand[subregionId] < 0):
gcore.warning(
_("Subregion {sub} has negative numbers"
" of newly developed cells, changing to zero".format(
sub=subregionId)))
demand[subregionId][demand[subregionId] < 0] = 0
if coeff[method][0] < 0:
# couldn't establish reliable population-area
# project by number of developed pixels in analyzed period
range_developed = table_developed[subregionId][
-1] - table_developed[subregionId][0]
range_times = observed_times[-1] - observed_times[0]
dev_per_step = math.ceil(range_developed / float(range_times))
# this assumes demand is projected yearly
demand[subregionId].fill(dev_per_step if dev_per_step > 0 else 0)
gcore.warning(
_("For subregion {sub} population and development are inversely proportional,"
" demand will be interpolated based on prior change in development only."
.format(sub=subregionId)))
# draw
if plot:
ax = fig.add_subplot(n_plots, n_plots, i)
ax.set_title("{sid}, RMSE: {rmse:.3f}".format(sid=subregionId,
rmse=rmse[method]))
ax.set_xlabel('population')
ax.set_ylabel('developed cells')
# plot known points
x = np.array(observed_popul[subregionId])
y = np.array(table_developed[subregionId])
ax.plot(x, y, marker='o', linestyle='', markersize=8)
# plot predicted curve
x_pred = np.linspace(
np.min(x),
np.max(np.array(population_for_simulated_times[subregionId])),
30)
cf = coeff[method]
if method == 'linear':
line = x_pred * cf[0] + cf[1]
label = "$y = {c:.3f} + {m:.3f} x$".format(m=cf[0], c=cf[1])
elif method == 'logarithmic':
line = np.log(x_pred) * cf[0] + cf[1]
label = "$y = {c:.3f} + {m:.3f} \ln(x)$".format(m=cf[0],
c=cf[1])
elif method == 'exponential':
line = np.exp(x_pred * cf[0] + cf[1])
label = "$y = {c:.3f} e^{{{m:.3f}x}}$".format(m=cf[0],
c=np.exp(cf[1]))
elif method == 'exp_approach':
line = exp_approach(x_pred / magn, *cf) * magn
label = "$y = (1 - e^{{-{A:.3f}(x-{B:.3f})}}) + {C:.3f}$".format(
A=cf[0], B=cf[1], C=cf[2])
elif method == 'logarithmic2':
line = logarithmic2(x_pred / magn, *cf) * magn
label = "$y = {A:.3f} + {B:.3f} \ln(x-{C:.3f})$".format(
A=cf[0], B=cf[1], C=cf[2])
ax.plot(x_pred, line, label=label)
ax.plot(simulated[method],
predicted[method],
linestyle='',
marker='o',
markerfacecolor='None')
plt.legend(loc=0)
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30)
if plot:
plt.tight_layout()
fig.savefig(plot)
# write demand
with open(options['demand'], 'w') as f:
header = observed_popul.dtype.names # the order is kept here
header = [header[0]
] + [sub for sub in header[1:] if sub in subregionIds]
f.write(sep.join(header))
f.write('\n')
i = 0
for time in simulation_times[1:]:
f.write(str(int(time)))
f.write(sep)
# put 0 where there are more counties but are not in region
for sub in header[1:]: # to keep order of subregions
f.write(str(int(demand[sub][i])))
if sub != header[-1]:
f.write(sep)
f.write('\n')
i += 1
def main():
strds = options["strds"]
out_name = options["output"]
if options["weight"] == '':
method = None
else:
method = options["weight"]
where = options["where"]
sep = separator(options["separator"])
if flags['p'] and not options["splittingday"]:
gscript.fatal(_("'p' flag required to set also 'splittingday' option"))
elif flags['p'] and options["splittingday"] and out_name == '-':
gscript.fatal(_("'output' option is required with 'p' flag"))
if flags['k'] and flags['p']:
gscript.fatal(_("It is not possible to use 'k' and 'p' flag together"))
elif flags['k'] and not method:
rkappa = True
elif flags['k'] and method:
gscript.message(_("If method is different from 'no' it is not possible"
" to use r.kappa"))
rkappa = _load_skll()
else:
rkappa = _load_skll()
tgis.init()
# We need a database interface
dbif = tgis.SQLDatabaseInterfaceConnection()
dbif.connect()
sp = tgis.open_old_stds(strds, "strds", dbif)
maps = sp.get_registered_maps_as_objects(where, "start_time", None)
if maps is None:
gscript.fatal(_("Space time raster dataset {st} seems to be "
"empty".format(st=strds)))
return 1
if flags['p']:
before, after = _split_maps(maps, options["splittingday"])
_kappa_pixel(before, after, out_name, method, gscript.overwrite())
return
mapnames = [mapp.get_name() for mapp in maps]
if not rkappa:
if out_name != '-':
fi = open(out_name, 'w')
else:
fi = sys.stdout
for i1 in range(len(mapnames)):
for i2 in range(i1 + 1, len(mapnames)):
map1 = mapnames[i1]
map2 = mapnames[i2]
if map1 != map2:
if not rkappa:
fi.write("{}-{}{}{}\n".format(map1, map2, sep,
_kappa_skll(map1, map2,
flags['l'],
method)))
else:
if out_name != '-':
fi = open("{}_{}_{}".format(out_name, map1, map2), 'w')
else:
fi = sys.stdout
fi.write("{}".format(_kappa_grass(map1, map2)))
if out_name != '-':
fi.close()
if not rkappa:
fi.close()
gscript.message(_("All data have analyzed"))
def main():
global tmp
fs = separator(options["separator"])
threeD = flags["z"]
prog = "v.in.lines"
if threeD:
do3D = "z"
else:
do3D = ""
tmp = grass.tempfile()
# set up input file
if options["input"] == "-":
infile = None
inf = sys.stdin
else:
infile = options["input"]
if not os.path.exists(infile):
grass.fatal(_("Unable to read input file <%s>") % infile)
grass.debug("input file=[%s]" % infile)
if not infile:
# read from stdin and write to tmpfile (v.in.mapgen wants a real file)
outf = file(tmp, "w")
for line in inf:
if len(line.lstrip()) == 0 or line[0] == "#":
continue
outf.write(line.replace(fs, " "))
outf.close()
runfile = tmp
else:
# read from a real file
if fs == " ":
runfile = infile
else:
inf = file(infile)
outf = file(tmp, "w")
for line in inf:
if len(line.lstrip()) == 0 or line[0] == "#":
continue
outf.write(line.replace(fs, " "))
inf.close()
outf.close()
runfile = tmp
# check that there are at least two columns (three if -z is given)
inf = file(runfile)
for line in inf:
if len(line.lstrip()) == 0 or line[0] == "#":
continue
numcols = len(line.split())
break
inf.close()
if (do3D and numcols < 3) or (not do3D and numcols < 2):
grass.fatal(_("Not enough data columns. (incorrect fs setting?)"))
grass.run_command("v.in.mapgen", flags="f" + do3D, input=runfile, output=options["output"])
def main():
check_addon_installed("r.object.geometry", fatal=True)
dev_start = options["development_start"]
dev_end = options["development_end"]
only_file = flags["l"]
nprocs = int(options["nprocs"])
patches_per_subregion = flags["s"]
if not only_file:
repeat = int(options["repeat"])
compactness_means = [
float(each) for each in options["compactness_mean"].split(",")
]
compactness_ranges = [
float(each) for each in options["compactness_range"].split(",")
]
discount_factors = [
float(each) for each in options["discount_factor"].split(",")
]
patches_file = options["patch_sizes"]
threshold = float(options["patch_threshold"])
sep = gutils.separator(options["separator"])
# v.clean removes size <= threshold, we want to keep size == threshold
threshold -= 1e-6
# compute cell size
region = gcore.region()
res = (region["nsres"] + region["ewres"]) / 2.0
coeff = float(gcore.parse_command("g.proj", flags="g")["meters"])
cell_size = res * res * coeff * coeff
tmp_name = "tmp_futures_calib_" + str(os.getpid()) + "_"
global TMP
orig_patch_diff = tmp_name + "orig_patch_diff"
TMP.append(orig_patch_diff)
tmp_clump = tmp_name + "tmp_clump"
TMP.append(tmp_clump)
if patches_per_subregion:
tmp_cat_clump = tmp_name + "tmp_cat_clump"
TMP.append(tmp_cat_clump)
gcore.message(_("Analyzing original patches..."))
diff_development(dev_start, dev_end, options["subregions"],
orig_patch_diff)
data = write_data = patch_analysis(orig_patch_diff, threshold, tmp_clump)
if patches_per_subregion:
subregions_data = patch_analysis_per_subregion_parallel(
orig_patch_diff,
options["subregions"],
threshold,
tmp_clump,
tmp_name,
nprocs,
)
# if there is just one column, write the previous analysis result
if len(subregions_data.keys()) > 1:
write_data = subregions_data
write_patches_file(write_data, cell_size, patches_file, sep)
if only_file:
return
area, perimeter = data.T
compact = compactness(area, perimeter)
# area histogram
area = area / cell_size
bin_width = (
1.0 # automatic ways to determine bin width do not perform well in this case
)
hist_bins_area_orig = int(np.ptp(area) / bin_width)
hist_range_area_orig = (np.min(area), np.max(area))
histogram_area_orig, _edges = np.histogram(area,
bins=hist_bins_area_orig,
range=hist_range_area_orig,
density=True)
histogram_area_orig = histogram_area_orig * 100 # to get percentage for readability
# compactness histogram
bin_width = 0.1
hist_bins_compactness_orig = int(np.ptp(compact) / bin_width)
hist_range_compactness_orig = (np.min(compact), np.max(compact))
histogram_compactness_orig, _edges = np.histogram(
compact,
bins=hist_bins_compactness_orig,
range=hist_range_compactness_orig,
density=True,
)
histogram_compactness_orig = (histogram_compactness_orig * 100
) # to get percentage for readability
seed = int(options["random_seed"])
count = 0
proc_count = 0
queue_list = []
proc_list = []
num_all = len(compactness_means) * len(compactness_ranges) * len(
discount_factors)
with open(options["calibration_results"], "w") as f:
for com_mean in compactness_means:
for com_range in compactness_ranges:
for discount_factor in discount_factors:
count += 1
q = Queue()
p = Process(
target=run_one_combination,
args=(
count,
num_all,
repeat,
seed,
dev_start,
com_mean,
com_range,
discount_factor,
patches_file,
options,
threshold,
hist_bins_area_orig,
hist_range_area_orig,
hist_bins_compactness_orig,
hist_range_compactness_orig,
cell_size,
histogram_area_orig,
histogram_compactness_orig,
tmp_name,
q,
),
)
p.start()
queue_list.append(q)
proc_list.append(p)
proc_count += 1
seed += 1
if proc_count == nprocs or count == num_all:
for i in range(proc_count):
proc_list[i].join()
data = queue_list[i].get()
if not data:
continue
f.write(",".join([
str(data["input_discount_factor"]),
str(data["area_distance"]),
str(data["input_compactness_mean"]),
str(data["input_compactness_range"]),
str(data["compactness_distance"]),
]))
f.write("\n")
f.flush()
proc_count = 0
proc_list = []
queue_list = []
# compute combined normalized error
process_calibration(options["calibration_results"])
