def write2struct(elines, opts): msgr = get_msgr() ktype = opts["struct_kind_turbine"] kcol = opts["struct_column_kind"] pname = opts["struct"] pname, vmapset = pname.split("@") if "@" in pname else (pname, "") with VectorTopo(pname, mapset=vmapset, layer=int(opts["struct_layer"]), mode="r") as vect: if "el_comp_exc" not in vect.table.columns: vect.table.columns.add("el_comp_exc", "double precision") ename, emapset = elines.split("@") if "@" in elines else (elines, "") with VectorTopo(ename, mapset=emapset, mode="r") as electro: for line in vect: if line.attrs[kcol] == ktype: plant_id = line.attrs["plant_id"] line.attrs["el_comp_exc"] = 0.0 for eline in electro: if plant_id == eline.attrs["plant_id"]: msgr.message(plant_id) cost = ((eline.attrs["comp_cost_sum"] + eline.attrs["exc_cost_sum"]) if eline.attrs["comp_cost_sum"] else 0) line.attrs["el_comp_exc"] = cost electro.rewind() break vect.table.conn.commit()
def mdebug(level, msg='', extra=None): """Debug decorators for class methods. :param level: the debug level :type level: int :param msg: Debug message :type msg: str :param extra: Function that return a string :type msg: func """ msgr = get_msgr() def decorator(method): @wraps(method) def wrapper(self, *args, **kargs): sargs = ', ' + ' , '.join([repr(a) for a in args]) if args else '' skargs = (' , '.join(['%s=%r' % (k, v) for k, v in kargs.items()]) if kargs else '') opts = "%s%s%s" % (sargs, ',' if sargs and skargs else '', skargs) dmsg = "%s.%s(self%s): %s %s" % ( self.__class__.__name__, method.__name__, opts, msg, extra(self, *args, **kargs) if extra else '') msgr.debug(level, dmsg) return method(self, *args, **kargs) return wrapper return decorator
def get_new_points(points, lines, output, maxdist=50): skipped = [] ovwr = gcore.overwrite() msgr = get_msgr() points, pmset = points.split('@') if '@' in points else (points, '') lines, lmset = lines.split('@') if '@' in lines else (lines, '') with VectorTopo(points, mapset=pmset, mode='r') as pts: cols = pts.table.columns.items() if pts.table else None with VectorTopo(lines, mapset=lmset, mode='r') as lns: with VectorTopo(output, mode='w', tab_cols=cols, overwrite=ovwr) as out: for pnt in pts: line = lns.find['by_point'].geo(pnt, maxdist=maxdist) if line is None: msg = ("Not found any line in the radius of %.2f " "for the point with cat: %d. The point " "will be skipped!") msgr.warning(msg % (maxdist, pnt.cat)) skipped.append(pnt.cat) continue # find the new point newpnt, dist, _, _ = line.distance(pnt) # get the attributes attrs = (None if pnt.attrs is None else pnt.attrs.values()[1:]) # write the new point in the new vector map out.write(newpnt, attrs) # save the changes on the output table out.table.conn.commit()
def extract_training(vect, tvect, tlayer): """Assign a class to all the areas that contained, are contained or intersect a training vector""" msgr = get_msgr() tname, tmset = tvect.split('@') if '@' in tvect else (tvect, '') vname, vmset = vect.split('@') if '@' in vect else (vect, '') with VectorTopo(tname, tmset, mode='r') as trn: with VectorTopo(vname, vmset, mode='r') as vct: layer_num, layer_name = get_layer_num_name(vct, tlayer) # instantiate the area objects trn_area = Area(c_mapinfo=trn.c_mapinfo) seg_area = Area(c_mapinfo=vct.c_mapinfo) n_areas = trn.number_of('areas') # check/remove/create a new table table, create_link = make_new_table(vct, layer_name, force=True) find_lines(table, [l for l in trn.viter('lines')], vct) # find and save all the segments find_area(table, trn.viter('areas', idonly=True), trn_area, seg_area, n_areas, vct) check_balance(table, trn.table) if create_link: msgr.message(_("Connect the new table to the vector map...")) with Vector(vect, mode='rw') as seg: link = Link(layer_num, name=layer_name, table=table.name) seg.dblinks.add(link) seg.build()
def check_compute_drain_stream(drain, stream, dtm, threshold=100000): """ Compute the stream and drain map with r.watersheld """ msgr = get_msgr() if (not(drain) or not(stream)): drain = 'drainage' stream = 'stream' gcore.run_command('r.watershed', elevation=dtm, threshold=threshold, drainage=drain, stream=stream) else: info1 = gcore.parse_command('r.info', flags='e', map=drain) info2 = gcore.parse_command('r.info', flags='e', map=stream) #pdb.set_trace() in1 = '\"%s\"' % (dtm) in2 = '\"%s\"' % (dtm) if ((in1 != info1['source1'] or (info1['description'] != '\"generated by r.watershed\"'))): warn = ("%s map not generated " "by r.watershed starting from %s") % (drain, dtm) msgr.warning(warn) if ((in2 != info2['source1'] or (info2['description'] != '\"generated by r.watershed\"'))): warn = ("%s map not generated " "by r.watershed starting from %s") % (stream, dtm) msgr.warning(warn) return drain, stream
def check_compute_basin_stream(basins, stream, dtm, threshold): """ Compute the stream and basin map with r.watersheld """ msgr = get_msgr() if (not(basins) or not(stream)): pid = os.getpid() basins = "tmprgreen_%i_basins" % pid stream = "tmprgreen_%i_stream" % pid gcore.run_command('r.watershed', elevation=dtm, threshold=threshold, basin=basins, stream=stream) else: info1 = gcore.parse_command('r.info', flags='e', map=basins) info2 = gcore.parse_command('r.info', flags='e', map=stream) #pdb.set_trace() in1 = '\"%s\"' % (dtm) if ((in1 != info1['source1'] or (info1['description'] != '\"generated by r.watershed\"'))): warn = ("%s map not generated " "by r.watershed starting from %s") % (basins, dtm) msgr.warning(warn) if ((in1 != info2['source1'] or (info2['description'] != '\"generated by r.watershed\"'))): warn = ("%s map not generated " "by r.watershed starting from %s") % (stream, dtm) msgr.warning(warn) return basins, stream
def write2struct(elines, opts): msgr = get_msgr() ktype = opts['struct_kind_turbine'] kcol = opts['struct_column_kind'] pname = opts['struct'] pname, vmapset = pname.split('@') if '@' in pname else (pname, '') with VectorTopo(pname, mapset=vmapset, layer=int(opts['struct_layer']), mode='r') as vect: if 'el_comp_exc' not in vect.table.columns: vect.table.columns.add('el_comp_exc', 'double precision') ename, emapset = elines.split('@') if '@' in elines else (elines, '') with VectorTopo(ename, mapset=emapset, mode='r') as electro: for line in vect: if line.attrs[kcol] == ktype: plant_id = line.attrs['plant_id'] line.attrs['el_comp_exc'] = 0. for eline in electro: if plant_id == eline.attrs['plant_id']: msgr.message(plant_id) cost = ((eline.attrs['comp_cost_sum'] + eline.attrs['exc_cost_sum']) if eline.attrs['comp_cost_sum'] else 0) line.attrs['el_comp_exc'] = cost electro.rewind() break vect.table.conn.commit()
def recursive_plant( args, range_plant, distance, start, end, rank, cat, line, plants, count, p_max, ): msgr = get_msgr() count = count + 1 #import ipdb; ipdb.set_trace() msgr.message("\n%i\n" % cat) #import pdb; pdb.set_trace() res = check_plant(args, range_plant, distance, start, end, rank, cat, line, count, p_max) if res: plants.append(res[0]) plants = check_segments(args, range_plant, distance, res[1], start, end, rank, cat, line, plants, count, p_max) return plants
def mdebug(level, msg='', extra=None): """Debug decorators for class methods. :param level: the debug level :type level: int :param msg: Debug message :type msg: str :param extra: Function that return a string :type msg: func """ msgr = get_msgr() def decorator(method): @wraps(method) def wrapper(self, *args, **kargs): sargs = ', ' + ' , '.join([repr(a) for a in args]) if args else '' skargs = (' , '.join(['%s=%r' % (k, v) for k, v in kargs.items()]) if kargs else '') opts = "%s%s%s" % (sargs, ',' if sargs and skargs else '', skargs) dmsg = "%s.%s(self%s): %s %s" % (self.__class__.__name__, method.__name__, opts, msg, extra(self, *args, **kargs) if extra else '') msgr.debug(level, dmsg) return method(self, *args, **kargs) return wrapper return decorator
def _init_tgis_message_interface(raise_on_error=False): """Initiate the global message interface :param raise_on_error: If True raise a FatalError exception in case of a fatal error, call sys.exit(1) otherwise """ global message_interface if message_interface is None: message_interface = messages.get_msgr(raise_on_error=raise_on_error)
def _init_tgis_message_interface(raise_on_error=False): """Initiate the global mesage interface :param raise_on_error: If True raise a FatalError exception in case of a fatal error, call sys.exit(1) otherwise """ global message_interface from grass.pygrass import messages message_interface = messages.get_msgr(raise_on_error=raise_on_error)
def find_lines(table, trn, seg): """Update the lines' table using the boundaries of the training areas""" msgr = get_msgr() sql = UPDATE.format(tname=table.name, cat=table.key) boxlist = BoxList() n_bounds = len(trn) cur = table.conn.cursor() for i, bound in enumerate(trn): msgr.percent(i, n_bounds, 1) alist = seg.find['by_box'].areas(bound.bbox(), boxlist) update_lines(bound, alist, cur, sql) table.conn.commit()
def economic2segment(economic, segment, basename='eco_', eco_layer=1, seg_layer=1, eco_pid='plant_id', seg_pid='plant_id', function=max_NPV, exclude=None): exclude = exclude if exclude else [] with VectorTopo(economic, mode='r') as eco: select_pids = 'SELECT {pid} FROM {tname} GROUP BY {pid};' etab = eco.table exclude.extend((etab.key, eco_pid)) ecols = [c for c in etab.columns if c not in exclude] cpids = etab.execute(select_pids.format(pid=eco_pid, tname=etab.name)) pids = list(cpids) # transform the cursor to a list cpids.close() # close cursor otherwise: dblock error... msgr = get_msgr() with VectorTopo(segment, mode='r') as seg: stab = seg.table scols = set(stab.columns.names()) # create the new columns if needed ucols = [] msgr.message('Check if column from economic already exists') #import ipdb; ipdb.set_trace() for col in ecols: column = basename + col if column not in scols: stab.columns.add(column, etab.columns[col]) ucols.append(column) for pid, in pids: print('%10s: ' % pid, end='') select_cats = 'SELECT {cat} FROM {tname} WHERE {cpid} LIKE "{pid}"' ecats = list(etab.execute(select_cats.format(cat=etab.key, tname=etab.name, cpid=eco_pid, pid=pid))) print('structures found, ', end='') ec0, ec1 = ecats[0][0], ecats[1][0] l0 = eco.cat(int(ec0), 'lines', layer=1)[0] l1 = eco.cat(int(ec1), 'lines', layer=1)[0] eattrs = function(l0, l1).attrs scats, = list(stab.execute(select_cats.format(cat=stab.key, tname=stab.name, cpid=seg_pid, pid=pid))) if len(scats) != 1: import ipdb; ipdb.set_trace() print('segment found, ', end='') # TODO: this is not efficient should be done in one step # to avoid to call several time the db update sattr = seg.cat(int(scats[0]), 'lines', layer=1)[0].attrs for ecol, scol in zip(ecols, ucols): sattr[scol] = str(eattrs[ecol]) print('segment updated!') stab.conn.commit() print('Finish')
def check_multilines(vector): vector, mset = vector.split("@") if "@" in vector else (vector, "") msgr = get_msgr() vec = VectorTopo(vector, mapset=mset, mode="r") vec.open("r") info = gcore.parse_command("v.category", input=vector, option="print") for i in info.keys(): vec.cat(int(i), "lines", 1) # if i == '28': # import ipdb; ipdb.set_trace() if len(vec.cat(int(i), "lines", 1)) > 1: # import ipdb; ipdb.set_trace() warn = "Multilines for the same category %s" % i msgr.warning(warn) vec.close()
def check_multilines(vector): vector, mset = vector.split('@') if '@' in vector else (vector, '') msgr = get_msgr() vec = VectorTopo(vector, mapset=mset, mode='r') vec.open("r") info = gcore.parse_command('v.category', input=vector, option='print') for i in info.keys(): vec.cat(int(i), 'lines', 1) # if i == '28': # import ipdb; ipdb.set_trace() if len(vec.cat(int(i), 'lines', 1)) > 1: # import ipdb; ipdb.set_trace() warn = ("Multilines for the same category %s" % i) msgr.warning(warn) vec.close()
def check_balance(table, trntab): """Checking the balance between different training classes.""" msg = _('Checking the balance between different training classes.') msgr = get_msgr() msgr.message(msg) chk_balance = ("SELECT class, count(*) as num_of_segments " "FROM {tname} " "GROUP BY class ORDER BY num_of_segments;") res = table.execute(chk_balance.format(tname=table.name)) cl_sql = "SELECT cat, class FROM {tname} ORDER BY cat;" clss = dict(trntab.execute(cl_sql.format(tname=trntab.name))) for cls, num in res.fetchall(): clname = clss.get(cls, str(cls)) msgr.message( " - %s (%d): %d" % (clname if clname else repr(clname), cls if cls else 0, num))
def dtm_corr(dtm, river, dtm_corr, lake=None): """Compute a new DTM by applying a corrective factor close to the river network. Output of r.green.watershed will be coherent with the river network """ pid = os.getpid() msgr = get_msgr() info = gcore.parse_command("g.region", flags="pgm") if lake: tmp_network = "tmprgreen_%i_network" % pid inputs = "%s,%s" % (lake, river) gcore.run_command("v.patch", input=inputs, output=tmp_network) river = tmp_network msgr.warning("The DTM will be temporarily modified") distance = [ float(info["nsres"]), float(info["nsres"]) * 1.5, float(info["nsres"]) * 3, ] pat = "tmprgreen_%i_" % pid for i, val in enumerate(distance): output = "%sbuff_%i" % (pat, i) gcore.run_command("v.buffer", input=river, output=output, distance=val) gcore.run_command( "v.to.rast", input=output, output=output, use="val", value=val, overwrite=True, ) command = ("%s_c = if(isnull(%s),0,%s)") % (output, output, output) mapcalc(command, overwrite=True) command = ("%s = if(%s,%s-%sbuff_0_c-%sbuff_1_c-%sbuff_2_c)") % ( dtm_corr, dtm, dtm, pat, pat, pat, ) mapcalc(command, overwrite=True)
def find_area(table, trn_ids, trn_area, seg_area, n_areas, seg): """Update the lines' table using the training areas""" msgr = get_msgr() cur = table.conn.cursor() msgr.message(_("Finding areas...")) sql = UPDATE.format(tname=table.name, cat=table.key) boxlist = BoxList() res = [] for i, trn_id in enumerate(trn_ids): msgr.percent(i, n_areas, 1) trn_area.id = trn_id trn_area.read() bblist = seg.find['by_box'].areas(trn_area.boundary.bbox(), boxlist, bboxlist_only=True) res.append( np.array(update_areas(trn_area, seg_area, bblist.ids, cur, sql))) table.conn.commit()
def compute_river_discharge(drain, stream, string, **kwargs): """ Given a stream network and drainage map it computes a rester with the the given resolution with the area in km2 of the upper basin for each pixel (bas_area) and a statistic on the bas_area for another series of raster if q_spec string=sum if piedmont case kwargs-> a=a, dtm=dtm and string=mean """ msgr = get_msgr() info = gcore.parse_command("r.info", flags="g", map=stream) raster_out = {} for name, value in kwargs.items(): raster_out[name] = raster2numpy(stream) bas_area = raster2numpy(stream) river_comp = raster2compressM(stream).tocoo() count = 0 for i, j in zip(river_comp.row, river_comp.col): count = count + 1 msgr.message("\n %i \n" % count) p_x, p_y = get_coo(stream, i, j) # pdb.set_trace() coo = "%f, %f" % (p_x, p_y) gcore.run_command( "r.water.outlet", overwrite=True, input=drain, output="area_temp", coordinates=coo, ) for name, value in kwargs.items(): formula = "temp = if(not(area_temp),0, %s)" % (value) # pdb.set_trace() mapcalc(formula, overwrite=True) # pdb.set_trace() info = gcore.parse_command("r.univar", map="temp", flags="g") # pdb.set_trace() raster_out[name][i][j] = float(info[string]) bas_area[i][j] = area_of_watershed("area_temp") # pdb.set_trace() return raster_out, bas_area
def make_new_table(vct, tname, cols=COLS, force=None): """Check/remove/create a new table""" msgr = get_msgr() force = overwrite() if force is None else force create_link = True # make a new table table = Table(tname, vct.table.conn) if table.exist(): if any([table.name == l.table_name for l in vct.dblinks]): create_link = False msg = _("Table <%s> already exist and will be removed.") msgr.warning(msg % table.name) table.drop(force=force) table.create(cols) # fill the new table with the segment cats slct = vct.table.filters.select(vct.table.key) cur = vct.table.execute(slct.get_sql()) table.insert(((cat[0], None) for cat in cur), many=True) table.conn.commit() return table, create_link
def check_raster_or_landuse(opts, params): """Return a list of raster name, i f necessary the rasters are generated using the file with the rules to convert a landuse map to the raster that it is needed. """ msgr = get_msgr() rasters = [] for par in params: if opts[par]: rasters.append(opts[par]) elif opts['rules_%s' % par] and opts['landuse']: output = rname(par) msg = 'Creating: {out} using the rules: {rul}' msgr.verbose(msg.format(out=output, rul='rules_%s' % par)) r.reclass(input=opts['landuse'], output=output, rules=opts['rules_%s' % par]) rasters.append(output) else: msg = '{par} or rule_{par} are required' raise ParameterError(msg.format(par=par)) return rasters
def main(options, flags): TMPRAST, TMPVECT, DEBUG = [], [], False atexit.register(cleanup, raster=TMPRAST, vector=TMPVECT, debug=DEBUG) elevation = options['elevation'] river = options['river'] # raster discharge = options['discharge'] # vec len_plant = float(options['len_plant']) len_min = float(options['len_min']) distance = float(options['distance']) efficiency = float(options['efficiency']) output_plant = options['output_plant'] output_point = options['output_point'] if options['p_max']: p_max = float(options['p_max']) else: p_max = None p_min = float(options['p_min']) DEBUG = flags['d'] c = flags['c'] msgr = get_msgr() #import ipdb; ipdb.set_trace() if c: msgr.message("\Clean rivers\n") TMPVECT = [("tmprgreen_%i_cleanb" % os.getpid())] pid = os.getpid() dissolve_lines(river, "tmprgreen_%i_cleanb" % os.getpid()) river = "tmprgreen_%i_cleanb" % pid # number of cell of the river # range for the solution msgr.message("\Loop on the category of segments\n") range_plant = (len_min, len_plant) plants = find_segments(river, discharge, elevation, range_plant, distance, p_max) # add l_min if output_point: write_points(plants, output_point, efficiency, p_min) write_plants(plants, output_plant, efficiency, p_min)
def fill_discharge_tot(bas, discharge_n, stream_n): """ Fill the total discharge for the basin b by knowing the relation among basins thank to the ID_all attribute in the object Basin """ msgr = get_msgr() warn = ("%i") % bas.ID ttt = sorted(discharge_n[stream_n == bas.ID]) # import ipdb; ipdb.set_trace() # bas.discharge_tot = 0.0 ttt = ttt[~np.isnan(ttt)] # FIXME: take the second bgger value to avoid to take the value of # another catchment, it is not so elegant # the low value is the upper part of the basin and the greater # the closure point if len(ttt) > 1 and not (math.isnan(ttt[-2])): bas.discharge_tot = float(ttt[-2]) else: bas.discharge_tot = 0.0 warn = ("No value for the river ID %i, discharge set to 0") % bas.ID msgr.warning(warn)
def check_compute_drain_stream(drain, stream, dtm, threshold=100000): """ Compute the stream and drain map with r.watersheld """ msgr = get_msgr() if not (drain) or not (stream): drain = "drainage" stream = "stream" gcore.run_command( "r.watershed", elevation=dtm, threshold=threshold, drainage=drain, stream=stream, ) else: info1 = gcore.parse_command("r.info", flags="e", map=drain) info2 = gcore.parse_command("r.info", flags="e", map=stream) # pdb.set_trace() in1 = '"%s"' % (dtm) in2 = '"%s"' % (dtm) if in1 != info1["source1"] or (info1["description"] != '"generated by r.watershed"'): warn = ("%s map not generated " "by r.watershed starting from %s") % ( drain, dtm, ) msgr.warning(warn) if in2 != info2["source1"] or (info2["description"] != '"generated by r.watershed"'): warn = ("%s map not generated " "by r.watershed starting from %s") % ( stream, dtm, ) msgr.warning(warn) return drain, stream
def rpatch_map( raster, mapset, mset_str, bbox_list, overwrite=False, start_row=0, start_col=0, prefix="", ): """Patch raster using a bounding box list to trim the raster.""" # Instantiate the RasterRow input objects rast = RasterRow(prefix + raster, mapset) with RasterRow(name=raster, mapset=mset_str % (0, 0), mode="r") as rtype: rast.open("w", mtype=rtype.mtype, overwrite=overwrite) msgr = get_msgr() rasts = [] mrast = 0 nrows = len(bbox_list) for row, rbbox in enumerate(bbox_list): rrasts = [] max_rasts = [] for col in range(len(rbbox)): msgr.percent(row, nrows, 1) rrasts.append( RasterRow(name=raster, mapset=mset_str % (start_row + row, start_col + col))) rrasts[-1].open("r") mrast += rrasts[-1].info.max + 1 max_rasts.append(mrast) rasts.append(rrasts) rpatch_row(rast, rrasts, rbbox, max_rasts) for rst in rrasts: rst.close() del rst rast.close()
def main(opt, flg): # import functions which depend on sklearn only after parser run from ml_functions import (balance, explorer_clsfiers, run_classifier, optimize_training, explore_SVC, plot_grid) from features import importances, tocsv msgr = get_msgr() indexes = None vect = opt['vector'] vtraining = opt['vtraining'] if opt['vtraining'] else None scaler, decmp = None, None vlayer = opt['vlayer'] if opt['vlayer'] else vect + '_stats' tlayer = opt['tlayer'] if opt['tlayer'] else vect + '_training' rlayer = opt['rlayer'] if opt['rlayer'] else vect + '_results' labels = extract_classes(vtraining, 1) pprint(labels) if opt['scalar']: scapar = opt['scalar'].split(',') from sklearn.preprocessing import StandardScaler scaler = StandardScaler(with_mean='with_mean' in scapar, with_std='with_std' in scapar) if opt['decomposition']: dec, params = (opt['decomposition'].split('|') if '|' in opt['decomposition'] else (opt['decomposition'], '')) kwargs = ({k: v for k, v in (p.split('=') for p in params.split(','))} if params else {}) load_decompositions() decmp = DECMP[dec](**kwargs) # if training extract training if vtraining and flg['e']: msgr.message("Extract training from: <%s> to <%s>." % (vtraining, vect)) extract_training(vect, vtraining, tlayer) flg['n'] = True if flg['n']: msgr.message("Save arrays to npy files.") save2npy(vect, vlayer, tlayer, fcats=opt['npy_cats'], fcols=opt['npy_cols'], fdata=opt['npy_data'], findx=opt['npy_index'], fclss=opt['npy_tclasses'], ftdata=opt['npy_tdata']) # define the classifiers to use/test if opt['pyclassifiers'] and opt['pyvar']: # import classifiers to use mycls = imp.load_source("mycls", opt['pyclassifiers']) classifiers = getattr(mycls, opt['pyvar']) else: from ml_classifiers import CLASSIFIERS classifiers = CLASSIFIERS # Append the SVC classifier if opt['svc_c'] and opt['svc_gamma']: from sklearn.svm import SVC svc = {'name': 'SVC', 'classifier': SVC, 'kwargs': {'C': float(opt['svc_c']), 'gamma': float(opt['svc_gamma']), 'kernel': opt['svc_kernel']}} classifiers.append(svc) # extract classifiers from pyindx if opt['pyindx']: indexes = [i for i in get_indexes(opt['pyindx'])] classifiers = [classifiers[i] for i in indexes] num = int(opt['n_training']) if opt['n_training'] else None # load fron npy files Xt = np.load(opt['npy_tdata']) Yt = np.load(opt['npy_tclasses']) cols = np.load(opt['npy_cols']) # Define rules to substitute NaN, Inf, posInf, negInf values rules = {} for key in ('nan', 'inf', 'neginf', 'posinf'): if opt[key]: rules[key] = get_rules(opt[key]) pprint(rules) # Substitute (skip cat column) Xt, rules_vals = substitute(Xt, rules, cols[1:]) Xtoriginal = Xt # scale the data if scaler: msgr.message("Scaling the training data set.") scaler.fit(Xt, Yt) Xt = scaler.transform(Xt) # decompose data if decmp: msgr.message("Decomposing the training data set.") decmp.fit(Xt) Xt = decmp.transform(Xt) # Feature importances with forests of trees if flg['f']: np.save('training_transformed.npy', Xt) importances(Xt, Yt, cols[1:], csv=opt['imp_csv'], img=opt['imp_fig'], # default parameters to save the matplotlib figure **dict(dpi=300, transparent=False, bbox_inches='tight')) # optimize the training set if flg['o']: ind_optimize = (int(opt['pyindx_optimize']) if opt['pyindx_optimize'] else 0) cls = classifiers[ind_optimize] msgr.message("Find the optimum training set.") best, Xbt, Ybt = optimize_training(cls, Xt, Yt, labels, #{v: k for k, v in labels.items()}, scaler, decmp, num=num, maxiterations=1000) msg = " - save the optimum training data set to: %s." msgr.message(msg % opt['npy_btdata']) np.save(opt['npy_btdata'], Xbt) msg = " - save the optimum training classes set to: %s." msgr.message(msg % opt['npy_btclasses']) np.save(opt['npy_btclasses'], Ybt) # balance the data if flg['b']: msg = "Balancing the training data set, each class have <%d> samples." msgr.message(msg % num) Xbt, Ybt = balance(Xt, Yt, num) else: if not flg['o']: Xbt = (np.load(opt['npy_btdata']) if os.path.isfile(opt['npy_btdata']) else Xt) Ybt = (np.load(opt['npy_btclasses']) if os.path.isfile(opt['npy_btclasses']) else Yt) # scale the data if scaler: msgr.message("Scaling the training data set.") scaler.fit(Xbt, Ybt) Xt = scaler.transform(Xt) Xbt = scaler.transform(Xbt) if flg['d']: C_range = [float(c) for c in opt['svc_c_range'].split(',') if c] gamma_range = [float(g) for g in opt['svc_gamma_range'].split(',') if g] kernel_range = [str(s) for s in opt['svc_kernel_range'].split(',') if s] poly_range = [int(i) for i in opt['svc_poly_range'].split(',') if i] allkwargs = dict(C=C_range, gamma=gamma_range, kernel=kernel_range, degree=poly_range) kwargs = {} for k in allkwargs: if allkwargs[k]: kwargs[k] = allkwargs[k] msgr.message("Exploring the SVC domain.") grid = explore_SVC(Xbt, Ybt, n_folds=5, n_jobs=int(opt['svc_n_jobs']), **kwargs) import pickle krnlstr = '_'.join(s for s in opt['svc_kernel_range'].split(',') if s) pkl = open('grid%s.pkl' % krnlstr, 'w') pickle.dump(grid, pkl) pkl.close() # pkl = open('grid.pkl', 'r') # grid = pickle.load(pkl) # pkl.close() plot_grid(grid, save=opt['svc_img']) # test the accuracy of different classifiers if flg['t']: # test different classifiers msgr.message("Exploring different classifiers.") msgr.message("cls_id cls_name mean max min std") res = explorer_clsfiers(classifiers, Xt, Yt, labels=labels, indexes=indexes, n_folds=5, bv=flg['v'], extra=flg['x']) # TODO: sort(order=...) is working only in the terminal, why? #res.sort(order='mean') with open(opt['csv_test_cls'], 'w') as csv: csv.write(tocsv(res)) if flg['c']: # classify data = np.load(opt['npy_data']) indx = np.load(opt['npy_index']) # Substitute using column values data, dummy = substitute(data, rules, cols[1:]) Xt = data[indx] if scaler: msgr.message("Scaling the training data set.") scaler.fit(Xt, Yt) Xt = scaler.transform(Xt) msgr.message("Scaling the whole data set.") data = scaler.transform(data) if decmp: msgr.message("Decomposing the training data set.") decmp.fit(Xt) Xt = decmp.transform(Xt) msgr.message("Decompose the whole data set.") data = decmp.transform(data) cats = np.load(opt['npy_cats']) np.save('data_filled_scaled.npy', data) tcols = [] for cls in classifiers: report = (open(opt['report_class'], "w") if opt['report_class'] else sys.stdout) run_classifier(cls, Xt, Yt, Xt, Yt, labels, data, report=report) tcols.append((cls['name'], 'INTEGER')) import pickle with open('classification_results.pkl', 'w') as res: pickle.dump(classifiers, res) #classifiers = pickle.load(res) msgr.message("Export the results to layer: <%s>" % str(rlayer)) export_results(vect, classifiers, cats, rlayer, vtraining, tcols, overwrite(), pkl='res.pkl', append=flg['a']) # res.close() if flg['r']: rules = ('\n'.join(['%d %s' % (k, v) for k, v in get_colors(vtraining).items()]) if vtraining else None) msgr.message("Export the layer with results to raster") with Vector(vect, mode='r') as vct: tab = vct.dblinks.by_name(rlayer).table() rasters = [c for c in tab.columns] rasters.remove(tab.key) v2rst = Module('v.to.rast') rclrs = Module('r.colors') for rst in rasters: v2rst(input=vect, layer=rlayer, type='area', use='attr', attrcolumn=rst.encode(), output=(opt['rst_names'] % rst).encode(), memory=1000, overwrite=overwrite()) if rules: rclrs(map=rst.encode(), rules='-', stdin_=rules)
def main(options, flags): ############################################################# # inizialitation ############################################################# pid = os.getpid() DEBUG = flags["d"] atexit.register(cleanup, pattern=("tmprgreen_%i*" % pid), debug=DEBUG) # TOD_add the possibilities to have q_points # required discharge = options["discharge"] dtm = options["elevation"] # basin potential basins = options["basins"] stream = options["stream"] # raster rivers = options["rivers"] # vec lakes = options["lakes"] # vec E = options["output"] threshold = options["threshold"] # existing plants # segments = options['segments'] # output_segm = options['output_segm'] # output_point = options['output_point'] # hydro = options['hydro'] # hydro_layer = options['hydro_layer'] # hydro_kind_intake = options['hydro_kind_intake'] # hydro_kind_turbine = options['hydro_kind_turbine'] # other = options['other'] # other_kind_intake = options['other_kind_intake'] # other_kind_turbine = options['other_kind_turbine'] # optional msgr = get_msgr() # info = gcore.parse_command('g.region', flags='m') # print info ############################################################# # check temporary raster if rivers: # cp the vector in the current mapset in order to clean it tmp_river = "tmprgreen_%i_river" % pid to_copy = "%s,%s" % (rivers, tmp_river) gcore.run_command("g.copy", vector=to_copy) rivers = tmp_river gcore.run_command("v.build", map=rivers) tmp_dtm_corr = "tmprgreen_%i_dtm_corr" % pid dtm_corr(dtm, rivers, tmp_dtm_corr, lakes) basins, stream = basin.check_compute_basin_stream( basins, stream, tmp_dtm_corr, threshold) else: basins, stream = basin.check_compute_basin_stream( basins, stream, dtm, threshold) perc_overlay = check_overlay_rr(discharge, stream) # pdb.set_trace() try: p = float(perc_overlay) if p < 90: warn = ("Discharge map doesn't overlay all the stream map." "It covers only the %s %% of rivers") % (perc_overlay) msgr.warning(warn) except ValueError: msgr.error("Could not convert data to a float") except: msgr.error("Unexpected error") msgr.message("\Init basins\n") # pdb.set_trace() ############################################################# # core ############################################################# basins_tot, inputs = basin.init_basins(basins) msgr.message("\nBuild the basin network\n") ############################################################# # build_network(stream, dtm, basins_tot) build relationship among basins # Identify the closure point with the r.neigbours module # if the difference betwwen the stream and the neighbours # is negative it means a new subsanin starts ############################################################# basin.build_network(stream, dtm, basins_tot) stream_n = raster2numpy(stream) discharge_n = raster2numpy(discharge) basin.fill_basins(inputs, basins_tot, basins, dtm, discharge_n, stream_n) ################################################################### # check if lakes and delate stream in lakes optional ################################################################### if lakes: remove_pixel_from_raster(lakes, stream) #################################################################### # write results #################################################################### # if not rivers or debug I write the result in the new vector stream msgr.message("\nWrite results\n") basin.write_results2newvec(stream, E, basins_tot, inputs) power2energy(E, "Etot_kW", 8760)
def main(opts, flgs): TMPRAST, TMPVECT, DEBUG = [], [], flgs["d"] atexit.register(cleanup, raster=TMPRAST, vector=TMPVECT, debug=DEBUG) OVW = gcore.overwrite() dtm = options["elevation"] river = options["river"] # raster discharge_current = options["discharge_current"] # vec discharge_natural = options["discharge_natural"] # vec mfd = options["mfd"] len_plant = options["len_plant"] len_min = options["len_min"] distance = options["distance"] output_plant = options["output_plant"] area = options["area"] buff = options["buff"] efficiency = options["efficiency"] DEBUG = flags["d"] points_view = options["points_view"] new_region = options["visibility_resolution"] final_vis = options["output_vis"] n_points = options["n_points"] p_min = options["p_min"] percentage = options["percentage"] msgr = get_msgr() # set the region info = gcore.parse_command("g.region", flags="m") if (info["nsres"] == 0) or (info["ewres"] == 0): msgr.warning("set region to elevation raster") gcore.run_command("g.region", raster=dtm) pid = os.getpid() if area: if float(buff): area_tmp = "tmp_buff_area_%05d" % pid gcore.run_command("v.buffer", input=area, output=area_tmp, distance=buff, overwrite=OVW) area = area_tmp TMPVECT.append(area) oriver = "tmp_river_%05d" % pid gcore.run_command( "v.overlay", flags="t", ainput=river, binput=area, operator="not", output=oriver, overwrite=OVW, ) river = oriver TMPVECT.append(oriver) if points_view: info_old = gcore.parse_command("g.region", flags="pg") set_new_region(new_region) pl, mset = points_view.split("@") if "@" in points_view else ( points_view, "") vec = VectorTopo(pl, mapset=mset, mode="r") vec.open("r") string = "0" for i, point in enumerate(vec): out = "tmp_visual_%05d_%03d" % (pid, i) gcore.run_command( "r.viewshed", input=dtm, output=out, coordinates=point.coords(), overwrite=OVW, memory=1000, flags="b", max_distance=4000, ) TMPRAST.append(out) # we use 4 km sice it the human limit string = string + ("+%s" % out) # import pdb; pdb.set_trace() tmp_final_vis = "tmp_final_vis_%05d" % pid formula = "%s = %s" % (tmp_final_vis, string) TMPRAST.append(tmp_final_vis) mapcalc(formula, overwrite=OVW) # change to old region set_old_region(info_old) TMPVECT.append(tmp_final_vis) gcore.run_command( "r.to.vect", flags="v", overwrite=OVW, input=tmp_final_vis, output=tmp_final_vis, type="area", ) if int(n_points) > 0: where = "cat<%s" % (n_points) else: where = "cat=0" gcore.run_command( "v.db.droprow", input=tmp_final_vis, where=where, output=final_vis, overwrite=OVW, ) tmp_river = "tmp_river2_%05d" % pid gcore.run_command( "v.overlay", flags="t", ainput=river, binput=final_vis, operator="not", output=tmp_river, overwrite=OVW, ) river = tmp_river TMPVECT.append(tmp_river) # import pdb; pdb.set_trace() tmp_disch = "tmp_discharge_%05d" % pid if mfd: formula = "%s=%s-%s" % (tmp_disch, discharge_current, mfd) mapcalc(formula, overwrite=OVW) TMPRAST.append(tmp_disch) discharge_current = tmp_disch elif discharge_natural: formula = "%s=%s-%s*%s/100.0" % ( tmp_disch, discharge_current, discharge_natural, percentage, ) mapcalc(formula, overwrite=OVW) formula = "%s=if(%s>0, %s, 0)" % (tmp_disch, tmp_disch, tmp_disch) mapcalc(formula, overwrite=True) TMPRAST.append(tmp_disch) discharge_current = tmp_disch gcore.run_command( "r.green.hydro.optimal", flags="c", discharge=discharge_current, river=river, elevation=dtm, len_plant=len_plant, output_plant=output_plant, distance=distance, len_min=len_min, efficiency=efficiency, p_min=p_min, ) power2energy(output_plant, "pot_power", float(options["n"])) print("r.green.hydro.recommended completed!")
def compute_losses(struct, options, percentage_losses, roughness_penstock, ks_derivation): # add necessary columns cols = [('diameter', 'DOUBLE'), ('losses', 'DOUBLE'), ('sg_losses', 'DOUBLE'), ('tot_losses', 'DOUBLE'), ('net_head', 'DOUBLE')] add_columns(struct, cols) # power column renamed? # extract intake id list_intakeid = list( set( struct.table.execute('SELECT intake_id FROM %s' % struct.table.name).fetchall())) theta = acos(1. / 6.) * 2 velocity = float(options['velocity_derivation']) # compute structure losses for line in struct: gross_head = float(line.attrs['gross_head']) discharge = float(line.attrs['discharge']) if gross_head <= 0 or discharge <= 0: # FIXME: it is not physical possible that it is <0 line.attrs['sg_losses'] = 0 line.attrs['gross_head'] = 0 line.attrs['losses'] = 0 else: length = line.length() losses = 0 if length > 0 and discharge > 0: if line.attrs['kind'] == 'penstock': diameter = line.attrs['diameter'] if not diameter: diameter = diam_pen(discharge, length, gross_head, percentage_losses, roughness_penstock) line.attrs['diameter'] = diameter length = (length**2.0 + gross_head**2.0)**0.5 if gross_head > length: msgr = get_msgr() msgr.warning("To check length of penstock," "gross head greater than " "length") import ipdb ipdb.set_trace() losses = losses_Colebrooke(discharge, length, diameter, roughness_penstock) # when you compute alpha (see manual) the lenght of the # line is the real length and not the projection # for the blend we use the formula sen2(alpha)+sen4(alpha/2) # TODO: add the possibility to insert the coefficient for the # singolar losses in the table of the structure # TODO: add singolar loss in function of thevelocity of # derivation channel line.attrs['sg_losses'] = singular_losses( gross_head, length, discharge, diameter) elif line.attrs['kind'] == 'conduct': diameter = line.attrs['diameter'] if not diameter: # diameter formula: d = sqrt(4 * Q / (v * pi)) # with: # - d as diameter # - Q as discharge # - v as velocity (v == 1) diameter = ((4 * discharge) / (velocity * pi))**(0.5) line.attrs['diameter'] = diameter losses = losses_Strickler(discharge, length, diameter, theta, velocity, ks_derivation) else: losses = 0 line.attrs['losses'] = losses # in [m] # TODO: fix as function of velocity # TODO: check when gross_head>length not physically # save the changes struct.table.conn.commit() # TODO: make it more readable/pythonic sql0 = "SELECT losses FROM %s WHERE (intake_id=%i AND side='option0');" sql1 = "SELECT losses FROM %s WHERE (intake_id=%i AND side='option1');" msgr = get_msgr() for i in list_intakeid: struct.rewind() totallosses0 = 0 totallosses1 = 0 bothlosses0 = list( struct.table.execute(sql0 % (struct.table.name, i[0])).fetchall()) if (bothlosses0[0][0] and bothlosses0[1][0]): totallosses0 = bothlosses0[0][0] + bothlosses0[1][0] bothlosses1 = list( struct.table.execute(sql1 % (struct.table.name, i[0])).fetchall()) if (bothlosses1[0][0] and bothlosses1[1][0]): totallosses1 = bothlosses1[0][0] + bothlosses1[1][0] for line in struct: sing_losses = line.attrs['sg_losses'] if (line.attrs['intake_id'] == i[0] and line.attrs['side'] == 'option0' and line.attrs['kind'] == 'penstock'): line.attrs['tot_losses'] = totallosses0 + sing_losses tot_losses = float(line.attrs['tot_losses']) line.attrs['net_head'] = max( 0.0, line.attrs['gross_head'] - tot_losses) # net_head = float(line.attrs['net_head']) if tot_losses > line.attrs['gross_head']: msgr.warning( ("Losses greater than gross_head, %i" % (line.cat))) if (line.attrs['intake_id'] == i[0] and line.attrs['side'] == 'option1' and line.attrs['kind'] == 'penstock'): line.attrs['tot_losses'] = totallosses1 + sing_losses tot_losses = float(line.attrs['tot_losses']) line.attrs['net_head'] = max( 0.0, line.attrs['gross_head'] - tot_losses) if tot_losses > line.attrs['gross_head']: msgr.warning( ("Losses greater than gross_head, %i" % (line.cat))) # net_head = float(line.attrs['net_head']) if line.attrs['kind'] == 'conduct': line.attrs['tot_losses'] = line.attrs['net_head'] = 0 struct.table.conn.commit() return list_intakeid
def wrapper(self, *args, **kargs): if self.is_open(): return method(self, *args, **kargs) else: get_msgr().warning(_("The map is close!"))
def main(options, flags): TMPRAST = [] DEBUG = True if flags['d'] else False atexit.register(cleanup, raster=TMPRAST, debug=DEBUG) elevation = options['elevation'] plant = options['plant'] output_struct = options['output_struct'] input_struct = options['input_struct'] output_plant = options['output_plant'] percentage_losses = float(options['percentage_losses']) roughness_penstock = float(options['roughness_penstock']) ks_derivation = float(options['ks_derivation']) turbine_folder = (options['turbine_folder'] if options['turbine_folder'] else os.path.join(os.path.abspath('.'), 'turbines')) turbine_list = (options['turbine_list'] if options['turbine_list'] else os.path.join(os.path.abspath('.'), 'turbines', 'list.txt')) efficiency_shaft = float(options['efficiency_shaft']) efficiency_alt = float(options['efficiency_alt']) efficiency_transf = float(options['efficiency_transf']) #import ipdb; ipdb.set_trace() #c = flags['c'] msgr = get_msgr() #import ipdb; ipdb.set_trace() TMPRAST.extend(['new_river', 'buff_area']) if not gcore.overwrite(): for m in TMPRAST: if gcore.find_file(m)['name']: msgr.fatal(_("Temporary raster map %s exists") % (m)) # FIXME:check if it works for vectors if options['output_point']: conv_segpoints(plant, options['output_point']) # FIXME: gross_head coherent with plants # set the opions to execute the r.green.hydro.structure struct_opts = dict(elevation=elevation, plant=plant, output_struct=output_struct, ndigits=options['ndigits'], contour=options['contour'], overwrite=gcore.overwrite()) if options['resolution']: struct_opts['resolution'] = options['resolution'] ## -------------------------------------------------------------------------- if output_struct: gcore.run_command('r.green.hydro.structure', **struct_opts) gcore.run_command('v.build', map=output_struct) else: output_struct = input_struct # FIXME: check the structure of the input file ## -------------------------------------------------------------------------- gcore.run_command('g.copy', vector=(plant, output_plant)) with VectorTopo(output_struct, mode='rw') as struct: list_intakeid = compute_losses(struct, options, percentage_losses, roughness_penstock, ks_derivation) compute_power(struct, list_intakeid, turbine_list, turbine_folder, efficiency_shaft, efficiency_alt, efficiency_transf) with VectorTopo(output_plant, mode='rw') as out, \ VectorTopo(output_struct, mode='r') as struct: cols = [('tot_losses', 'DOUBLE'), ('net_head', 'DOUBLE'), ('e_global', 'DOUBLE'), ('power', 'DOUBLE')] add_columns(out, cols) scols = 'tot_losses', 'net_head', 'e_global', 'power' wherecond = 'plant_id={!r} AND kind LIKE {!r} AND max_power LIKE {!r}' for seg in out: where = wherecond.format(str(seg.attrs['plant_id']), 'penstock', 'yes') sqlcode = struct.table.filters.select( ','.join(scols)).where(where).get_sql() svalues = struct.table.execute(sqlcode).fetchone() if svalues: for col, value in zip(scols, svalues): if value: seg.attrs[col] = value out.table.conn.commit() power2energy(output_plant, 'power', float(options['n']))
def main(): elev = options['input'] output = options['output'] n_dir = int(options['ndir']) global TMP_NAME, CLEANUP if options['basename']: TMP_NAME = options['basename'] CLEANUP = False colorized_output = options['colorized_output'] colorize_color = options['color_table'] if colorized_output: color_raster_tmp = TMP_NAME + "_color_raster" else: color_raster_tmp = None color_raster_type = options['color_source'] color_input = options['color_input'] if color_raster_type == 'color_input' and not color_input: gcore.fatal(_("Provide raster name in color_input option")) if color_raster_type != 'color_input' and color_input: gcore.fatal( _("The option color_input is not needed" " when not using it as source for color")) # this would be needed only when no value would allowed if not color_raster_type and color_input: color_raster_type = 'color_input' # enable for convenience if color_raster_type == 'aspect' \ and colorize_color \ and colorize_color not in ['default', 'aspectcolr']: gcore.warning( _("Using possibly inappropriate color table <{}>" " for aspect".format(colorize_color))) horizon_step = 360. / n_dir msgr = get_msgr() # checks if there are already some maps old_maps = _get_horizon_maps() if old_maps: if not gcore.overwrite(): CLEANUP = False msgr.fatal( _("You have to first check overwrite flag or remove" " the following maps:\n" "{names}").format(names=','.join(old_maps))) else: msgr.warning( _("The following maps will be overwritten: {names}").format( names=','.join(old_maps))) if not gcore.overwrite() and color_raster_tmp: check_map_name(color_raster_tmp) try: params = {} if options['maxdistance']: params['maxdistance'] = options['maxdistance'] gcore.run_command('r.horizon', elevation=elev, step=horizon_step, output=TMP_NAME, flags='d', **params) new_maps = _get_horizon_maps() if flags['o']: msgr.message(_("Computing openness ...")) expr = '{out} = 1 - (sin({first}) '.format(first=new_maps[0], out=output) for horizon in new_maps[1:]: expr += '+ sin({name}) '.format(name=horizon) expr += ") / {n}.".format(n=len(new_maps)) else: msgr.message(_("Computing skyview factor ...")) expr = '{out} = 1 - (sin( if({first} < 0, 0, {first}) ) '.format( first=new_maps[0], out=output) for horizon in new_maps[1:]: expr += '+ sin( if({name} < 0, 0, {name}) ) '.format( name=horizon) expr += ") / {n}.".format(n=len(new_maps)) grast.mapcalc(exp=expr) gcore.run_command('r.colors', map=output, color='grey') except CalledModuleError: msgr.fatal( _("r.horizon failed to compute horizon elevation " "angle maps. Please report this problem to developers.")) return 1 if colorized_output: if color_raster_type == 'slope': gcore.run_command('r.slope.aspect', elevation=elev, slope=color_raster_tmp) elif color_raster_type == 'aspect': gcore.run_command('r.slope.aspect', elevation=elev, aspect=color_raster_tmp) elif color_raster_type == 'dxy': gcore.run_command('r.slope.aspect', elevation=elev, dxy=color_raster_tmp) elif color_raster_type == 'color_input': color_raster_tmp = color_input else: color_raster_tmp = elev # don't modify user's color table for inputs if colorize_color \ and color_raster_type not in ['input', 'color_input']: rcolors_flags = '' if flags['n']: rcolors_flags += 'n' gcore.run_command('r.colors', map=color_raster_tmp, color=colorize_color, flags=rcolors_flags) gcore.run_command('r.shade', shade=output, color=color_raster_tmp, output=colorized_output) grast.raster_history(colorized_output) grast.raster_history(output) return 0