def import_stds(input, output, directory, title=None, descr=None, location=None, link=False, exp=False, overr=False, create=False, stds_type="strds", base=None, set_current_region=False, memory=300): """Import space time datasets of type raster and vector :param input: Name of the input archive file :param output: The name of the output space time dataset :param directory: The extraction directory :param title: The title of the new created space time dataset :param descr: The description of the new created space time dataset :param location: The name of the location that should be created, maps are imported into this location :param link: Switch to link raster maps instead importing them :param exp: Extend location extents based on new dataset :param overr: Override projection (use location's projection) :param create: Create the location specified by the "location" parameter and exit. Do not import the space time datasets. :param stds_type: The type of the space time dataset that should be imported :param base: The base name of the new imported maps, it will be extended using a numerical index. :param memory: Cache size for raster rows, used in r.in.gdal """ global raise_on_error old_state = gscript.raise_on_error gscript.set_raise_on_error(True) # Check if input file and extraction directory exits if not os.path.exists(input): gscript.fatal( _("Space time raster dataset archive <%s> not found") % input) if not create and not os.path.exists(directory): gscript.fatal(_("Extraction directory <%s> not found") % directory) tar = tarfile.open(name=input, mode='r') # Check for important files msgr = get_tgis_message_interface() msgr.message( _("Checking validity of input file (size: %0.1f MB). Make take a while..." % (os.path.getsize(input) / (1024 * 1024.0)))) members = tar.getnames() # Make sure that the basenames of the files are used for comparison member_basenames = [os.path.basename(name) for name in members] if init_file_name not in member_basenames: gscript.fatal(_("Unable to find init file <%s>") % init_file_name) if list_file_name not in member_basenames: gscript.fatal(_("Unable to find list file <%s>") % list_file_name) if proj_file_name not in member_basenames: gscript.fatal( _("Unable to find projection file <%s>") % proj_file_name) msgr.message(_("Extracting data...")) tar.extractall(path=directory) tar.close() # We use a new list file name for map registration new_list_file_name = list_file_name + "_new" # Save current working directory path old_cwd = os.getcwd() # Switch into the data directory os.chdir(directory) # Check projection information if not location: temp_name = gscript.tempfile() temp_file = open(temp_name, "w") proj_name = os.path.abspath(proj_file_name) # We need to convert projection strings generated # from other programs than g.proj into # new line format so that the grass file comparison function # can be used to compare the projections proj_name_tmp = temp_name + "_in_projection" proj_file = open(proj_name, "r") proj_content = proj_file.read() proj_content = proj_content.replace(" +", "\n+") proj_content = proj_content.replace("\t+", "\n+") proj_file.close() proj_file = open(proj_name_tmp, "w") proj_file.write(proj_content) proj_file.close() p = gscript.start_command("g.proj", flags="j", stdout=temp_file) p.communicate() temp_file.close() if not gscript.compare_key_value_text_files( temp_name, proj_name_tmp, sep="="): if overr: gscript.warning( _("Projection information does not match. " "Proceeding...")) else: diff = ''.join(gscript.diff_files(temp_name, proj_name)) gscript.warning( _("Difference between PROJ_INFO file of " "imported map and of current location:" "\n{diff}").format(diff=diff)) gscript.fatal( _("Projection information does not match. " "Aborting.")) # Create a new location based on the projection information and switch # into it old_env = gscript.gisenv() if location: try: proj4_string = open(proj_file_name, 'r').read() gscript.create_location(dbase=old_env["GISDBASE"], location=location, proj4=proj4_string) # Just create a new location and return if create: os.chdir(old_cwd) return except Exception as e: gscript.fatal( _("Unable to create location %(l)s. Reason: %(e)s") % { 'l': location, 'e': str(e) }) # Switch to the new created location try: gscript.run_command("g.mapset", mapset="PERMANENT", location=location, dbase=old_env["GISDBASE"]) except CalledModuleError: gscript.fatal(_("Unable to switch to location %s") % location) # create default database connection try: gscript.run_command("t.connect", flags="d") except CalledModuleError: gscript.fatal( _("Unable to create default temporal database " "in new location %s") % location) try: # Make sure the temporal database exists factory.init() fs = "|" maplist = [] mapset = get_current_mapset() list_file = open(list_file_name, "r") new_list_file = open(new_list_file_name, "w") # get number of lines to correctly form the suffix max_count = -1 for max_count, l in enumerate(list_file): pass max_count += 1 list_file.seek(0) # Read the map list from file line_count = 0 while True: line = list_file.readline() if not line: break line_list = line.split(fs) # The filename is actually the base name of the map # that must be extended by the file suffix filename = line_list[0].strip().split(":")[0] if base: mapname = "%s_%s" % ( base, gscript.get_num_suffix(line_count + 1, max_count)) mapid = "%s@%s" % (mapname, mapset) else: mapname = filename mapid = mapname + "@" + mapset row = {} row["filename"] = filename row["name"] = mapname row["id"] = mapid row["start"] = line_list[1].strip() row["end"] = line_list[2].strip() new_list_file.write("%s%s%s%s%s\n" % (mapname, fs, row["start"], fs, row["end"])) maplist.append(row) line_count += 1 list_file.close() new_list_file.close() # Read the init file fs = "=" init = {} init_file = open(init_file_name, "r") while True: line = init_file.readline() if not line: break kv = line.split(fs) init[kv[0]] = kv[1].strip() init_file.close() if "temporal_type" not in init or \ "semantic_type" not in init or \ "number_of_maps" not in init: gscript.fatal( _("Key words %(t)s, %(s)s or %(n)s not found in init" " file.") % { 't': "temporal_type", 's': "semantic_type", 'n': "number_of_maps" }) if line_count != int(init["number_of_maps"]): gscript.fatal(_("Number of maps mismatch in init and list file.")) format_ = "GTiff" type_ = "strds" if "stds_type" in init: type_ = init["stds_type"] if "format" in init: format_ = init["format"] if stds_type != type_: gscript.fatal( _("The archive file is of wrong space time dataset" " type")) # Check the existence of the files if format_ == "GTiff": for row in maplist: filename = row["filename"] + ".tif" if not os.path.exists(filename): gscript.fatal( _("Unable to find GeoTIFF raster file " "<%s> in archive.") % filename) elif format_ == "AAIGrid": for row in maplist: filename = row["filename"] + ".asc" if not os.path.exists(filename): gscript.fatal( _("Unable to find AAIGrid raster file " "<%s> in archive.") % filename) elif format_ == "GML": for row in maplist: filename = row["filename"] + ".xml" if not os.path.exists(filename): gscript.fatal( _("Unable to find GML vector file " "<%s> in archive.") % filename) elif format_ == "pack": for row in maplist: if type_ == "stvds": filename = str(row["filename"].split(":")[0]) + ".pack" else: filename = row["filename"] + ".pack" if not os.path.exists(filename): gscript.fatal( _("Unable to find GRASS package file " "<%s> in archive.") % filename) else: gscript.fatal(_("Unsupported input format")) # Check the space time dataset id = output + "@" + mapset sp = dataset_factory(type_, id) if sp.is_in_db() and gscript.overwrite() is False: gscript.fatal( _("Space time %(t)s dataset <%(sp)s> is already in" " the database. Use the overwrite flag.") % { 't': type_, 'sp': sp.get_id() }) # Import the maps if type_ == "strds": if format_ == "GTiff" or format_ == "AAIGrid": _import_raster_maps_from_gdal(maplist, overr, exp, location, link, format_, set_current_region, memory) if format_ == "pack": _import_raster_maps(maplist, set_current_region) elif type_ == "stvds": if format_ == "GML": _import_vector_maps_from_gml(maplist, overr, exp, location, link) if format_ == "pack": _import_vector_maps(maplist) # Create the space time dataset if sp.is_in_db() and gscript.overwrite() is True: gscript.info( _("Overwrite space time %(sp)s dataset " "<%(id)s> and unregister all maps.") % { 'sp': sp.get_new_map_instance(None).get_type(), 'id': sp.get_id() }) sp.delete() sp = sp.get_new_instance(id) temporal_type = init["temporal_type"] semantic_type = init["semantic_type"] relative_time_unit = None if temporal_type == "relative": if "relative_time_unit" not in init: gscript.fatal( _("Key word %s not found in init file.") % ("relative_time_unit")) relative_time_unit = init["relative_time_unit"] sp.set_relative_time_unit(relative_time_unit) gscript.verbose( _("Create space time %s dataset.") % sp.get_new_map_instance(None).get_type()) sp.set_initial_values(temporal_type=temporal_type, semantic_type=semantic_type, title=title, description=descr) sp.insert() # register the maps fs = "|" register_maps_in_space_time_dataset( type=sp.get_new_map_instance(None).get_type(), name=output, file=new_list_file_name, start="file", end="file", unit=relative_time_unit, dbif=None, fs=fs, update_cmd_list=False) os.chdir(old_cwd) except: raise # Make sure the location is switched back correctly finally: if location: # Switch to the old location try: gscript.run_command("g.mapset", mapset=old_env["MAPSET"], location=old_env["LOCATION_NAME"], gisdbase=old_env["GISDBASE"]) except CalledModuleError: grass.warning(_("Switching to original location failed")) gscript.set_raise_on_error(old_state)
def dataset_mapcalculator(inputs, output, type, expression, base, method, nprocs=1, register_null=False, spatial=False): """Perform map-calculations of maps from different space time raster/raster3d datasets, using a specific sampling method to select temporal related maps. A mapcalc expression must be provided to process the temporal selected maps. Temporal operators are available in addition to the r.mapcalc operators: Supported operators for relative and absolute time are: - td() - the time delta of the current interval in days and fractions of days or the unit in case of relative time - start_time() - The start time of the interval from the begin of the time series in days and fractions of days or the unit in case of relative time - end_time() - The end time of the current interval from the begin of the time series in days and fractions of days or the unit in case of relative time Supported operators for absolute time: - start_doy() - Day of year (doy) from the start time [1 - 366] - start_dow() - Day of week (dow) from the start time [1 - 7], the start of the week is monday == 1 - start_year() - The year of the start time [0 - 9999] - start_month() - The month of the start time [1 - 12] - start_week() - Week of year of the start time [1 - 54] - start_day() - Day of month from the start time [1 - 31] - start_hour() - The hour of the start time [0 - 23] - start_minute() - The minute of the start time [0 - 59] - start_second() - The second of the start time [0 - 59] - end_doy() - Day of year (doy) from the end time [1 - 366] - end_dow() - Day of week (dow) from the end time [1 - 7], the start of the week is monday == 1 - end_year() - The year of the end time [0 - 9999] - end_month() - The month of the end time [1 - 12] - end_week() - Week of year of the end time [1 - 54] - end_day() - Day of month from the end time [1 - 31] - end_hour() - The hour of the end time [0 - 23] - end_minute() - The minute of the end time [0 - 59] - end_second() - The minute of the end time [0 - 59] :param inputs: The names of the input space time raster/raster3d datasets :param output: The name of the extracted new space time raster(3d) dataset :param type: The type of the dataset: "raster" or "raster3d" :param expression: The r(3).mapcalc expression :param base: The base name of the new created maps in case a mapclac expression is provided :param method: The method to be used for temporal sampling :param nprocs: The number of parallel processes to be used for mapcalc processing :param register_null: Set this number True to register empty maps :param spatial: Check spatial overlap """ # We need a database interface for fast computation dbif = SQLDatabaseInterfaceConnection() dbif.connect() mapset = get_current_mapset() msgr = get_tgis_message_interface() input_name_list = inputs.split(",") first_input = open_old_stds(input_name_list[0], type, dbif) # All additional inputs in reverse sorted order to avoid # wrong name substitution input_name_list = input_name_list[1:] input_name_list.sort() input_name_list.reverse() input_list = [] for input in input_name_list: sp = open_old_stds(input, type, dbif) input_list.append(copy.copy(sp)) new_sp = check_new_stds(output, type, dbif, gscript.overwrite()) # Sample all inputs by the first input and create a sample matrix if spatial: msgr.message(_("Starting spatio-temporal sampling...")) else: msgr.message(_("Starting temporal sampling...")) map_matrix = [] id_list = [] sample_map_list = [] # First entry is the first dataset id id_list.append(first_input.get_name()) if len(input_list) > 0: has_samples = False for dataset in input_list: list = dataset.sample_by_dataset(stds=first_input, method=method, spatial=spatial, dbif=dbif) # In case samples are not found if not list and len(list) == 0: dbif.close() msgr.message(_("No samples found for map calculation")) return 0 # The fist entries are the samples map_name_list = [] if not has_samples: for entry in list: granule = entry["granule"] # Do not consider gaps if granule.get_id() is None: continue sample_map_list.append(granule) map_name_list.append(granule.get_name()) # Attach the map names map_matrix.append(copy.copy(map_name_list)) has_samples = True map_name_list = [] for entry in list: maplist = entry["samples"] granule = entry["granule"] # Do not consider gaps in the sampler if granule.get_id() is None: continue if len(maplist) > 1: msgr.warning(_("Found more than a single map in a sample " "granule. Only the first map is used for " "computation. Use t.rast.aggregate.ds to " "create synchronous raster datasets.")) # Store all maps! This includes non existent maps, # identified by id == None map_name_list.append(maplist[0].get_name()) # Attach the map names map_matrix.append(copy.copy(map_name_list)) id_list.append(dataset.get_name()) else: list = first_input.get_registered_maps_as_objects(dbif=dbif) if list is None: dbif.close() msgr.message(_("No maps registered in input dataset")) return 0 map_name_list = [] for map in list: map_name_list.append(map.get_name()) sample_map_list.append(map) # Attach the map names map_matrix.append(copy.copy(map_name_list)) # Needed for map registration map_list = [] if len(map_matrix) > 0: msgr.message(_("Starting mapcalc computation...")) count = 0 # Get the number of samples num = len(map_matrix[0]) # Parallel processing proc_list = [] proc_count = 0 # For all samples for i in range(num): count += 1 if count % 10 == 0: msgr.percent(count, num, 1) # Create the r.mapcalc statement for the current time step map_name = "{base}_{suffix}".format(base=base, suffix=gscript.get_num_suffix(count, num)) # Remove spaces and new lines expr = expression.replace(" ", "") # Check that all maps are in the sample valid_maps = True # Replace all dataset names with their map names of the # current time step for j in range(len(map_matrix)): if map_matrix[j][i] is None: valid_maps = False break # Substitute the dataset name with the map name expr = expr.replace(id_list[j], map_matrix[j][i]) # Proceed with the next sample if not valid_maps: continue # Create the new map id and check if the map is already # in the database map_id = map_name + "@" + mapset new_map = first_input.get_new_map_instance(map_id) # Check if new map is in the temporal database if new_map.is_in_db(dbif): if gscript.overwrite(): # Remove the existing temporal database entry new_map.delete(dbif) new_map = first_input.get_new_map_instance(map_id) else: msgr.error(_("Map <%s> is already in temporal database, " "use overwrite flag to overwrite")) continue # Set the time stamp if sample_map_list[i].is_time_absolute(): start, end = sample_map_list[i].get_absolute_time() new_map.set_absolute_time(start, end) else: start, end, unit = sample_map_list[i].get_relative_time() new_map.set_relative_time(start, end, unit) # Parse the temporal expressions expr = _operator_parser(expr, sample_map_list[0], sample_map_list[i]) # Add the output map name expr = "%s=%s" % (map_name, expr) map_list.append(new_map) msgr.verbose(_("Apply mapcalc expression: \"%s\"") % expr) # Start the parallel r.mapcalc computation if type == "raster": proc_list.append(Process(target=_run_mapcalc2d, args=(expr,))) else: proc_list.append(Process(target=_run_mapcalc3d, args=(expr,))) proc_list[proc_count].start() proc_count += 1 if proc_count == nprocs or proc_count == num or count == num: proc_count = 0 exitcodes = 0 for proc in proc_list: proc.join() exitcodes += proc.exitcode if exitcodes != 0: dbif.close() msgr.fatal(_("Error while mapcalc computation")) # Empty process list proc_list = [] # Register the new maps in the output space time dataset msgr.message(_("Starting map registration in temporal database...")) temporal_type, semantic_type, title, description = first_input.get_initial_values() new_sp = open_new_stds(output, type, temporal_type, title, description, semantic_type, dbif, gscript.overwrite()) count = 0 # collect empty maps to remove them empty_maps = [] # Insert maps in the temporal database and in the new space time # dataset for new_map in map_list: count += 1 if count % 10 == 0: msgr.percent(count, num, 1) # Read the map data new_map.load() # In case of a null map continue, do not register null maps if new_map.metadata.get_min() is None and \ new_map.metadata.get_max() is None: if not register_null: empty_maps.append(new_map) continue # Insert map in temporal database new_map.insert(dbif) new_sp.register_map(new_map, dbif) # Update the spatio-temporal extent and the metadata table entries new_sp.update_from_registered_maps(dbif) msgr.percent(1, 1, 1) # Remove empty maps if len(empty_maps) > 0: names = "" count = 0 for map in empty_maps: if count == 0: names += "%s" % (map.get_name()) else: names += ",%s" % (map.get_name()) count += 1 if type == "raster": gscript.run_command("g.remove", flags='f', type='rast', pattern=names, quiet=True) elif type == "raster3d": gscript.run_command("g.remove", flags='f', type='rast3d', pattern=names, quiet=True) dbif.close()
def dataset_mapcalculator(inputs, output, type, expression, base, method, nprocs=1, register_null=False, spatial=False): """Perform map-calculations of maps from different space time raster/raster3d datasets, using a specific sampling method to select temporal related maps. A mapcalc expression must be provided to process the temporal selected maps. Temporal operators are available in addition to the r.mapcalc operators: Supported operators for relative and absolute time are: - td() - the time delta of the current interval in days and fractions of days or the unit in case of relative time - start_time() - The start time of the interval from the begin of the time series in days and fractions of days or the unit in case of relative time - end_time() - The end time of the current interval from the begin of the time series in days and fractions of days or the unit in case of relative time Supported operators for absolute time: - start_doy() - Day of year (doy) from the start time [1 - 366] - start_dow() - Day of week (dow) from the start time [1 - 7], the start of the week is monday == 1 - start_year() - The year of the start time [0 - 9999] - start_month() - The month of the start time [1 - 12] - start_week() - Week of year of the start time [1 - 54] - start_day() - Day of month from the start time [1 - 31] - start_hour() - The hour of the start time [0 - 23] - start_minute() - The minute of the start time [0 - 59] - start_second() - The second of the start time [0 - 59] - end_doy() - Day of year (doy) from the end time [1 - 366] - end_dow() - Day of week (dow) from the end time [1 - 7], the start of the week is monday == 1 - end_year() - The year of the end time [0 - 9999] - end_month() - The month of the end time [1 - 12] - end_week() - Week of year of the end time [1 - 54] - end_day() - Day of month from the end time [1 - 31] - end_hour() - The hour of the end time [0 - 23] - end_minute() - The minute of the end time [0 - 59] - end_second() - The minute of the end time [0 - 59] :param inputs: The names of the input space time raster/raster3d datasets :param output: The name of the extracted new space time raster(3d) dataset :param type: The type of the dataset: "raster" or "raster3d" :param expression: The r(3).mapcalc expression :param base: The base name of the new created maps in case a mapclac expression is provided :param method: The method to be used for temporal sampling :param nprocs: The number of parallel processes to be used for mapcalc processing :param register_null: Set this number True to register empty maps :param spatial: Check spatial overlap """ # We need a database interface for fast computation dbif = SQLDatabaseInterfaceConnection() dbif.connect() mapset = get_current_mapset() msgr = get_tgis_message_interface() input_name_list = inputs.split(",") first_input = open_old_stds(input_name_list[0], type, dbif) # All additional inputs in reverse sorted order to avoid # wrong name substitution input_name_list = input_name_list[1:] input_name_list.sort() input_name_list.reverse() input_list = [] for input in input_name_list: sp = open_old_stds(input, type, dbif) input_list.append(copy.copy(sp)) new_sp = check_new_stds(output, type, dbif, gscript.overwrite()) # Sample all inputs by the first input and create a sample matrix if spatial: msgr.message(_("Starting spatio-temporal sampling...")) else: msgr.message(_("Starting temporal sampling...")) map_matrix = [] id_list = [] sample_map_list = [] # First entry is the first dataset id id_list.append(first_input.get_name()) if len(input_list) > 0: has_samples = False for dataset in input_list: list = dataset.sample_by_dataset(stds=first_input, method=method, spatial=spatial, dbif=dbif) # In case samples are not found if not list or len(list) == 0: dbif.close() msgr.message(_("No samples found for map calculation")) return 0 # The fist entries are the samples map_name_list = [] if not has_samples: for entry in list: granule = entry["granule"] # Do not consider gaps if granule.get_id() is None: continue sample_map_list.append(granule) map_name_list.append(granule.get_name()) # Attach the map names map_matrix.append(copy.copy(map_name_list)) has_samples = True map_name_list = [] for entry in list: maplist = entry["samples"] granule = entry["granule"] # Do not consider gaps in the sampler if granule.get_id() is None: continue if len(maplist) > 1: msgr.warning( _("Found more than a single map in a sample " "granule. Only the first map is used for " "computation. Use t.rast.aggregate.ds to " "create synchronous raster datasets.")) # Store all maps! This includes non existent maps, # identified by id == None map_name_list.append(maplist[0].get_name()) # Attach the map names map_matrix.append(copy.copy(map_name_list)) id_list.append(dataset.get_name()) else: list = first_input.get_registered_maps_as_objects(dbif=dbif) if list is None: dbif.close() msgr.message(_("No maps registered in input dataset")) return 0 map_name_list = [] for map in list: map_name_list.append(map.get_name()) sample_map_list.append(map) # Attach the map names map_matrix.append(copy.copy(map_name_list)) # Needed for map registration map_list = [] if len(map_matrix) > 0: msgr.message(_("Starting mapcalc computation...")) count = 0 # Get the number of samples num = len(map_matrix[0]) # Parallel processing proc_list = [] proc_count = 0 # For all samples for i in range(num): count += 1 msgr.percent(count, num, 10) # Create the r.mapcalc statement for the current time step map_name = "{base}_{suffix}".format(base=base, suffix=gscript.get_num_suffix( count, num)) # Remove spaces and new lines expr = expression.replace(" ", "") # Check that all maps are in the sample valid_maps = True # Replace all dataset names with their map names of the # current time step for j in range(len(map_matrix)): if map_matrix[j][i] is None: valid_maps = False break # Substitute the dataset name with the map name expr = expr.replace(id_list[j], map_matrix[j][i]) # Proceed with the next sample if not valid_maps: continue # Create the new map id and check if the map is already # in the database map_id = map_name + "@" + mapset new_map = first_input.get_new_map_instance(map_id) # Check if new map is in the temporal database if new_map.is_in_db(dbif): if gscript.overwrite(): # Remove the existing temporal database entry new_map.delete(dbif) new_map = first_input.get_new_map_instance(map_id) else: msgr.error( _("Map <%s> is already in temporal database, " "use overwrite flag to overwrite")) continue # Set the time stamp if sample_map_list[i].is_time_absolute(): start, end = sample_map_list[i].get_absolute_time() new_map.set_absolute_time(start, end) else: start, end, unit = sample_map_list[i].get_relative_time() new_map.set_relative_time(start, end, unit) # Parse the temporal expressions expr = _operator_parser(expr, sample_map_list[0], sample_map_list[i]) # Add the output map name expr = "%s=%s" % (map_name, expr) map_list.append(new_map) msgr.verbose(_("Apply mapcalc expression: \"%s\"") % expr) # Start the parallel r.mapcalc computation if type == "raster": proc_list.append(Process(target=_run_mapcalc2d, args=(expr, ))) else: proc_list.append(Process(target=_run_mapcalc3d, args=(expr, ))) proc_list[proc_count].start() proc_count += 1 if proc_count == nprocs or proc_count == num or count == num: proc_count = 0 exitcodes = 0 for proc in proc_list: proc.join() exitcodes += proc.exitcode if exitcodes != 0: dbif.close() msgr.fatal(_("Error while mapcalc computation")) # Empty process list proc_list = [] # Register the new maps in the output space time dataset msgr.message(_("Starting map registration in temporal database...")) temporal_type, semantic_type, title, description = first_input.get_initial_values( ) new_sp = open_new_stds(output, type, temporal_type, title, description, semantic_type, dbif, gscript.overwrite()) count = 0 # collect empty maps to remove them empty_maps = [] # Insert maps in the temporal database and in the new space time # dataset for new_map in map_list: count += 1 msgr.percent(count, num, 10) # Read the map data new_map.load() # In case of a null map continue, do not register null maps if new_map.metadata.get_min() is None and \ new_map.metadata.get_max() is None: if not register_null: empty_maps.append(new_map) continue # Insert map in temporal database new_map.insert(dbif) new_sp.register_map(new_map, dbif) # Update the spatio-temporal extent and the metadata table entries new_sp.update_from_registered_maps(dbif) # Remove empty maps if len(empty_maps) > 0: n_empty, n_tot = len(empty_maps), len(map_list) msgr.warning( _("Removing {}/{} ({}%) maps because empty...").format( n_empty, n_tot, n_empty * 100. / n_tot)) names = "" count = 0 for map in empty_maps: if count == 0: names += "%s" % (map.get_name()) else: names += ",%s" % (map.get_name()) count += 1 if type == "raster": gscript.run_command("g.remove", flags='f', type='raster', name=names, quiet=True) elif type == "raster3d": gscript.run_command("g.remove", flags='f', type='raster_3d', name=names, quiet=True) dbif.close()
def import_stds(input, output, directory, title=None, descr=None, location=None, link=False, exp=False, overr=False, create=False, stds_type="strds", base=None, set_current_region=False): """Import space time datasets of type raster and vector :param input: Name of the input archive file :param output: The name of the output space time dataset :param directory: The extraction directory :param title: The title of the new created space time dataset :param descr: The description of the new created space time dataset :param location: The name of the location that should be created, maps are imported into this location :param link: Switch to link raster maps instead importing them :param exp: Extend location extents based on new dataset :param overr: Override projection (use location's projection) :param create: Create the location specified by the "location" parameter and exit. Do not import the space time datasets. :param stds_type: The type of the space time dataset that should be imported :param base: The base name of the new imported maps, it will be extended using a numerical index. """ global raise_on_error old_state = gscript.raise_on_error gscript.set_raise_on_error(True) # Check if input file and extraction directory exits if not os.path.exists(input): gscript.fatal(_("Space time raster dataset archive <%s> not found") % input) if not create and not os.path.exists(directory): gscript.fatal(_("Extraction directory <%s> not found") % directory) tar = tarfile.open(name=input, mode='r') # Check for important files msgr = get_tgis_message_interface() msgr.message(_("Checking validity of input file (size: %0.1f MB). Make take a while..." % (os.path.getsize(input)/(1024*1024.0)))) members = tar.getnames() # Make sure that the basenames of the files are used for comparison member_basenames = [os.path.basename(name) for name in members] if init_file_name not in member_basenames: gscript.fatal(_("Unable to find init file <%s>") % init_file_name) if list_file_name not in member_basenames: gscript.fatal(_("Unable to find list file <%s>") % list_file_name) if proj_file_name not in member_basenames: gscript.fatal(_("Unable to find projection file <%s>") % proj_file_name) msgr.message(_("Extracting data...")) tar.extractall(path=directory) tar.close() # We use a new list file name for map registration new_list_file_name = list_file_name + "_new" # Save current working directory path old_cwd = os.getcwd() # Switch into the data directory os.chdir(directory) # Check projection information if not location: temp_name = gscript.tempfile() temp_file = open(temp_name, "w") proj_name = os.path.abspath(proj_file_name) # We need to convert projection strings generated # from other programms than g.proj into # new line format so that the grass file comparison function # can be used to compare the projections proj_name_tmp = temp_name + "_in_projection" proj_file = open(proj_name, "r") proj_content = proj_file.read() proj_content = proj_content.replace(" +", "\n+") proj_content = proj_content.replace("\t+", "\n+") proj_file.close() proj_file = open(proj_name_tmp, "w") proj_file.write(proj_content) proj_file.close() p = gscript.start_command("g.proj", flags="j", stdout=temp_file) p.communicate() temp_file.close() if not gscript.compare_key_value_text_files(temp_name, proj_name_tmp, sep="="): if overr: gscript.warning(_("Projection information does not match. " "Proceeding...")) else: diff = ''.join(gscript.diff_files(temp_name, proj_name)) gscript.warning(_("Difference between PROJ_INFO file of " "imported map and of current location:" "\n{diff}").format(diff=diff)) gscript.fatal(_("Projection information does not match. " "Aborting.")) # Create a new location based on the projection information and switch # into it old_env = gscript.gisenv() if location: try: proj4_string = open(proj_file_name, 'r').read() gscript.create_location(dbase=old_env["GISDBASE"], location=location, proj4=proj4_string) # Just create a new location and return if create: os.chdir(old_cwd) return except Exception as e: gscript.fatal(_("Unable to create location %(l)s. Reason: %(e)s") % {'l': location, 'e': str(e)}) # Switch to the new created location try: gscript.run_command("g.mapset", mapset="PERMANENT", location=location, dbase=old_env["GISDBASE"]) except CalledModuleError: gscript.fatal(_("Unable to switch to location %s") % location) # create default database connection try: gscript.run_command("t.connect", flags="d") except CalledModuleError: gscript.fatal(_("Unable to create default temporal database " "in new location %s") % location) try: # Make sure the temporal database exists factory.init() fs = "|" maplist = [] mapset = get_current_mapset() list_file = open(list_file_name, "r") new_list_file = open(new_list_file_name, "w") # get number of lines to correctly form the suffix max_count = -1 for max_count, l in enumerate(list_file): pass max_count += 1 list_file.seek(0) # Read the map list from file line_count = 0 while True: line = list_file.readline() if not line: break line_list = line.split(fs) # The filename is actually the base name of the map # that must be extended by the file suffix filename = line_list[0].strip().split(":")[0] if base: mapname = "%s_%s" % (base, gscript.get_num_suffix(line_count + 1, max_count)) mapid = "%s@%s" % (mapname, mapset) else: mapname = filename mapid = mapname + "@" + mapset row = {} row["filename"] = filename row["name"] = mapname row["id"] = mapid row["start"] = line_list[1].strip() row["end"] = line_list[2].strip() new_list_file.write("%s%s%s%s%s\n" % (mapname, fs, row["start"], fs, row["end"])) maplist.append(row) line_count += 1 list_file.close() new_list_file.close() # Read the init file fs = "=" init = {} init_file = open(init_file_name, "r") while True: line = init_file.readline() if not line: break kv = line.split(fs) init[kv[0]] = kv[1].strip() init_file.close() if "temporal_type" not in init or \ "semantic_type" not in init or \ "number_of_maps" not in init: gscript.fatal(_("Key words %(t)s, %(s)s or %(n)s not found in init" " file.") % {'t': "temporal_type", 's': "semantic_type", 'n': "number_of_maps"}) if line_count != int(init["number_of_maps"]): gscript.fatal(_("Number of maps mismatch in init and list file.")) format_ = "GTiff" type_ = "strds" if "stds_type" in init: type_ = init["stds_type"] if "format" in init: format_ = init["format"] if stds_type != type_: gscript.fatal(_("The archive file is of wrong space time dataset" " type")) # Check the existence of the files if format_ == "GTiff": for row in maplist: filename = row["filename"] + ".tif" if not os.path.exists(filename): gscript.fatal(_("Unable to find GeoTIFF raster file " "<%s> in archive.") % filename) elif format_ == "AAIGrid": for row in maplist: filename = row["filename"] + ".asc" if not os.path.exists(filename): gscript.fatal(_("Unable to find AAIGrid raster file " "<%s> in archive.") % filename) elif format_ == "GML": for row in maplist: filename = row["filename"] + ".xml" if not os.path.exists(filename): gscript.fatal(_("Unable to find GML vector file " "<%s> in archive.") % filename) elif format_ == "pack": for row in maplist: if type_ == "stvds": filename = str(row["filename"].split(":")[0]) + ".pack" else: filename = row["filename"] + ".pack" if not os.path.exists(filename): gscript.fatal(_("Unable to find GRASS package file " "<%s> in archive.") % filename) else: gscript.fatal(_("Unsupported input format")) # Check the space time dataset id = output + "@" + mapset sp = dataset_factory(type_, id) if sp.is_in_db() and gscript.overwrite() is False: gscript.fatal(_("Space time %(t)s dataset <%(sp)s> is already in" " the database. Use the overwrite flag.") % {'t': type_, 'sp': sp.get_id()}) # Import the maps if type_ == "strds": if format_ == "GTiff" or format_ == "AAIGrid": _import_raster_maps_from_gdal(maplist, overr, exp, location, link, format_, set_current_region) if format_ == "pack": _import_raster_maps(maplist, set_current_region) elif type_ == "stvds": if format_ == "GML": _import_vector_maps_from_gml( maplist, overr, exp, location, link) if format_ == "pack": _import_vector_maps(maplist) # Create the space time dataset if sp.is_in_db() and gscript.overwrite() is True: gscript.info(_("Overwrite space time %(sp)s dataset " "<%(id)s> and unregister all maps.") % {'sp': sp.get_new_map_instance(None).get_type(), 'id': sp.get_id()}) sp.delete() sp = sp.get_new_instance(id) temporal_type = init["temporal_type"] semantic_type = init["semantic_type"] relative_time_unit = None if temporal_type == "relative": if "relative_time_unit" not in init: gscript.fatal(_("Key word %s not found in init file.") % ("relative_time_unit")) relative_time_unit = init["relative_time_unit"] sp.set_relative_time_unit(relative_time_unit) gscript.verbose(_("Create space time %s dataset.") % sp.get_new_map_instance(None).get_type()) sp.set_initial_values(temporal_type=temporal_type, semantic_type=semantic_type, title=title, description=descr) sp.insert() # register the maps fs = "|" register_maps_in_space_time_dataset( type=sp.get_new_map_instance(None).get_type(), name=output, file=new_list_file_name, start="file", end="file", unit=relative_time_unit, dbif=None, fs=fs, update_cmd_list=False) os.chdir(old_cwd) except: raise # Make sure the location is switched back correctly finally: if location: # Switch to the old location try: gscript.run_command("g.mapset", mapset=old_env["MAPSET"], location=old_env["LOCATION_NAME"], gisdbase=old_env["GISDBASE"]) except CalledModuleError: grass.warning(_("Switching to original location failed")) gscript.set_raise_on_error(old_state)
def aggregate_by_topology(granularity_list, granularity, map_list, topo_list, basename, time_suffix, offset=0, method="average", nprocs=1, spatial=None, dbif=None, overwrite=False): """Aggregate a list of raster input maps with r.series :param granularity_list: A list of AbstractMapDataset objects. The temporal extents of the objects are used to build the spatio-temporal topology with the map list objects :param granularity: The granularity of the granularity list :param map_list: A list of RasterDataset objects that contain the raster maps that should be aggregated :param topo_list: A list of strings of topological relations that are used to select the raster maps for aggregation :param basename: The basename of the new generated raster maps :param time_suffix: Use the granularity truncated start time of the actual granule to create the suffix for the basename :param offset: Use a numerical offset for suffix generation (overwritten by time_suffix) :param method: The aggregation method of r.series (average,min,max, ...) :param nprocs: The number of processes used for parallel computation :param spatial: This indicates if the spatial topology is created as well: spatial can be None (no spatial topology), "2D" using west, east, south, north or "3D" using west, east, south, north, bottom, top :param dbif: The database interface to be used :param overwrite: Overwrite existing raster maps :return: A list of RasterDataset objects that contain the new map names and the temporal extent for map registration """ import grass.pygrass.modules as pymod import copy msgr = get_tgis_message_interface() dbif, connected = init_dbif(dbif) topo_builder = SpatioTemporalTopologyBuilder() topo_builder.build(mapsA=granularity_list, mapsB=map_list, spatial=spatial) # The module queue for parallel execution process_queue = pymod.ParallelModuleQueue(int(nprocs)) # Dummy process object that will be deep copied # and be put into the process queue r_series = pymod.Module("r.series", output="spam", method=[method], overwrite=overwrite, quiet=True, run_=False, finish_=False) g_copy = pymod.Module("g.copy", rast=['spam', 'spamspam'], quiet=True, run_=False, finish_=False) output_list = [] count = 0 for granule in granularity_list: msgr.percent(count, len(granularity_list), 1) count += 1 aggregation_list = [] if "equal" in topo_list and granule.equal: for map_layer in granule.equal: aggregation_list.append(map_layer.get_name()) if "contains" in topo_list and granule.contains: for map_layer in granule.contains: aggregation_list.append(map_layer.get_name()) if "during" in topo_list and granule.during: for map_layer in granule.during: aggregation_list.append(map_layer.get_name()) if "starts" in topo_list and granule.starts: for map_layer in granule.starts: aggregation_list.append(map_layer.get_name()) if "started" in topo_list and granule.started: for map_layer in granule.started: aggregation_list.append(map_layer.get_name()) if "finishes" in topo_list and granule.finishes: for map_layer in granule.finishes: aggregation_list.append(map_layer.get_name()) if "finished" in topo_list and granule.finished: for map_layer in granule.finished: aggregation_list.append(map_layer.get_name()) if "overlaps" in topo_list and granule.overlaps: for map_layer in granule.overlaps: aggregation_list.append(map_layer.get_name()) if "overlapped" in topo_list and granule.overlapped: for map_layer in granule.overlapped: aggregation_list.append(map_layer.get_name()) if aggregation_list: msgr.verbose( _("Aggregating %(len)i raster maps from %(start)s to" " %(end)s") % ({ "len": len(aggregation_list), "start": str(granule.temporal_extent.get_start_time()), "end": str(granule.temporal_extent.get_end_time()) })) if granule.is_time_absolute() is True and time_suffix is True: suffix = create_suffix_from_datetime( granule.temporal_extent.get_start_time(), granularity) else: suffix = gscript.get_num_suffix( count + int(offset), len(granularity_list) + int(offset)) output_name = "%s_%s" % (basename, suffix) map_layer = RasterDataset("%s@%s" % (output_name, get_current_mapset())) map_layer.set_temporal_extent(granule.get_temporal_extent()) if map_layer.map_exists() is True and overwrite is False: msgr.fatal( _("Unable to perform aggregation. Output raster " "map <%(name)s> exists and overwrite flag was " "not set" % ({ "name": output_name }))) output_list.append(map_layer) if len(aggregation_list) > 1: # Create the r.series input file filename = gscript.tempfile(True) file = open(filename, 'w') for name in aggregation_list: string = "%s\n" % (name) file.write(string) file.close() mod = copy.deepcopy(r_series) mod(file=filename, output=output_name) if len(aggregation_list) > 1000: mod(flags="z") process_queue.put(mod) else: mod = copy.deepcopy(g_copy) mod(rast=[aggregation_list[0], output_name]) process_queue.put(mod) if connected: dbif.close() msgr.percent(1, 1, 1) return output_list
def aggregate_by_topology(granularity_list, granularity, map_list, topo_list, basename, time_suffix, offset=0, method="average", nprocs=1, spatial=None, dbif=None, overwrite=False, file_limit=1000): """Aggregate a list of raster input maps with r.series :param granularity_list: A list of AbstractMapDataset objects. The temporal extents of the objects are used to build the spatio-temporal topology with the map list objects :param granularity: The granularity of the granularity list :param map_list: A list of RasterDataset objects that contain the raster maps that should be aggregated :param topo_list: A list of strings of topological relations that are used to select the raster maps for aggregation :param basename: The basename of the new generated raster maps :param time_suffix: Use the granularity truncated start time of the actual granule to create the suffix for the basename :param offset: Use a numerical offset for suffix generation (overwritten by time_suffix) :param method: The aggregation method of r.series (average,min,max, ...) :param nprocs: The number of processes used for parallel computation :param spatial: This indicates if the spatial topology is created as well: spatial can be None (no spatial topology), "2D" using west, east, south, north or "3D" using west, east, south, north, bottom, top :param dbif: The database interface to be used :param overwrite: Overwrite existing raster maps :param file_limit: The maximum number of raster map layers that should be opened at once by r.series :return: A list of RasterDataset objects that contain the new map names and the temporal extent for map registration """ import grass.pygrass.modules as pymod import copy msgr = get_tgis_message_interface() dbif, connected = init_dbif(dbif) topo_builder = SpatioTemporalTopologyBuilder() topo_builder.build(mapsA=granularity_list, mapsB=map_list, spatial=spatial) # The module queue for parallel execution process_queue = pymod.ParallelModuleQueue(int(nprocs)) # Dummy process object that will be deep copied # and be put into the process queue r_series = pymod.Module("r.series", output="spam", method=[method], overwrite=overwrite, quiet=True, run_=False, finish_=False) g_copy = pymod.Module("g.copy", raster=['spam', 'spamspam'], quiet=True, run_=False, finish_=False) output_list = [] count = 0 for granule in granularity_list: msgr.percent(count, len(granularity_list), 1) count += 1 aggregation_list = [] if "equal" in topo_list and granule.equal: for map_layer in granule.equal: aggregation_list.append(map_layer.get_name()) if "contains" in topo_list and granule.contains: for map_layer in granule.contains: aggregation_list.append(map_layer.get_name()) if "during" in topo_list and granule.during: for map_layer in granule.during: aggregation_list.append(map_layer.get_name()) if "starts" in topo_list and granule.starts: for map_layer in granule.starts: aggregation_list.append(map_layer.get_name()) if "started" in topo_list and granule.started: for map_layer in granule.started: aggregation_list.append(map_layer.get_name()) if "finishes" in topo_list and granule.finishes: for map_layer in granule.finishes: aggregation_list.append(map_layer.get_name()) if "finished" in topo_list and granule.finished: for map_layer in granule.finished: aggregation_list.append(map_layer.get_name()) if "overlaps" in topo_list and granule.overlaps: for map_layer in granule.overlaps: aggregation_list.append(map_layer.get_name()) if "overlapped" in topo_list and granule.overlapped: for map_layer in granule.overlapped: aggregation_list.append(map_layer.get_name()) if aggregation_list: msgr.verbose(_("Aggregating %(len)i raster maps from %(start)s to" " %(end)s") %({"len": len(aggregation_list), "start": str(granule.temporal_extent.get_start_time()), "end": str(granule.temporal_extent.get_end_time())})) if granule.is_time_absolute() is True and time_suffix is True: suffix = create_suffix_from_datetime(granule.temporal_extent.get_start_time(), granularity) else: suffix = gscript.get_num_suffix(count + int(offset), len(granularity_list) + int(offset)) output_name = "%s_%s" % (basename, suffix) map_layer = RasterDataset("%s@%s" % (output_name, get_current_mapset())) map_layer.set_temporal_extent(granule.get_temporal_extent()) if map_layer.map_exists() is True and overwrite is False: msgr.fatal(_("Unable to perform aggregation. Output raster " "map <%(name)s> exists and overwrite flag was " "not set" % ({"name": output_name}))) output_list.append(map_layer) if len(aggregation_list) > 1: # Create the r.series input file filename = gscript.tempfile(True) file = open(filename, 'w') for name in aggregation_list: string = "%s\n" % (name) file.write(string) file.close() mod = copy.deepcopy(r_series) mod(file=filename, output=output_name) if len(aggregation_list) > int(file_limit): msgr.warning(_("The limit of open fiels (%i) was "\ "reached (%i). The module r.series will "\ "be run with flag z, to avoid open "\ "files limit exceeding."%(int(file_limit), len(aggregation_list)))) mod(flags="z") process_queue.put(mod) else: mod = copy.deepcopy(g_copy) mod(raster=[aggregation_list[0], output_name]) process_queue.put(mod) process_queue.wait() if connected: dbif.close() msgr.percent(1, 1, 1) return output_list