def __init__(self, project_db, editor_version, project_name=None, datasets_db=None, constant_ps=True, is_lte=False): self.__abort = False base_path = os.path.dirname(project_db) rel_project_db = os.path.relpath(project_db, base_path) if datasets_db is None: conn = lib.open_db(project_db) if not lib.exists_table(conn, 'project_config'): sys.exit( 'No datasets database provided and the project_config table in your project database does not exist. Please provide either a datasets database file or an existing project database.' ) SetupProjectDatabase.init(project_db) try: config = Project_config.get() datasets_db = utils.full_path(project_db, config.reference_db) project_name = config.project_name except Project_config.DoesNotExist: sys.exit('Could not retrieve project configuration data.') rel_datasets_db = os.path.relpath(datasets_db, base_path) # Run updates if needed SetupProjectDatabase.init(project_db, datasets_db) config = Project_config.get() if config.editor_version in update_project.available_to_update: update_project.UpdateProject(project_db, editor_version, update_project_values=True) # Backup original db before beginning try: self.emit_progress(2, 'Backing up project database...') filename, file_extension = os.path.splitext(rel_project_db) bak_filename = filename + '_bak_' + time.strftime( '%Y%m%d-%H%M%S') + file_extension bak_dir = os.path.join(base_path, 'DatabaseBackups') if not os.path.exists(bak_dir): os.makedirs(bak_dir) backup_db_file = os.path.join(bak_dir, bak_filename) copyfile(project_db, backup_db_file) except IOError as err: sys.exit(err) self.emit_progress(5, 'Updating project settings...') config = Project_config.get( ) # Get again due to modification when updating config.imported_gis = False config.is_lte = is_lte config.save() api = GisImport(project_db, True, constant_ps, backup_db_file) api.insert_default()
def put(self): parser = reqparse.RequestParser() parser.add_argument('project_name', type=str, required=True, location='json') parser.add_argument('project_db', type=str, required=True, location='json') parser.add_argument('reference_db', type=str, required=True, location='json') args = parser.parse_args(strict=False) project_db = utils.sanitize(args['project_db']) reference_db = utils.sanitize(args['reference_db']) SetupProjectDatabase.init(project_db) try: m = Project_config.get() m.reference_db = utils.rel_path(project_db, args['reference_db']) m.project_name = args['project_name'] result = m.save() if result > 0: return 200 abort(400, message="Unable to update project configuration table.") except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def put(self, project_db): parser = reqparse.RequestParser() parser.add_argument('input_files_dir', type=str, required=False, location='json') parser.add_argument('input_files_last_written', type=str, required=False, location='json') args = parser.parse_args(strict=False) SetupProjectDatabase.init(project_db) try: m = Project_config.get() m.input_files_dir = utils.rel_path(project_db, args['input_files_dir']) m.input_files_last_written = args['input_files_last_written'] result = m.save() if result > 0: return 200 abort(400, message="Unable to update project configuration table.") except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def get(self, project_db): check_config(project_db) SetupProjectDatabase.init(project_db) try: m = Project_config.get() d = get_model_to_dict_dates(m, project_db) if not import_gis.is_supported_version( m.gis_version ) and not import_gis_legacy.is_supported_version(m.gis_version): abort( 400, message= "This version of SWAT+ Editor does not support QSWAT+ {uv}." .format(uv=m.gis_version)) d["has_ps"] = gis.Gis_points.select().where( (gis.Gis_points.ptype == 'P') | (gis.Gis_points.ptype == 'I')).count() > 0 d["has_res"] = gis.Gis_water.select().count() > 0 return d except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def get(self, project_db): check_config(project_db) SetupProjectDatabase.init(project_db) try: m = Project_config.get() d = get_model_to_dict_dates(m, project_db) if m.gis_version is not None and not import_gis.is_supported_version( m.gis_version ) and not import_gis_legacy.is_supported_version(m.gis_version): abort( 400, message= "This version of SWAT+ Editor does not support QSWAT+ {uv}." .format(uv=m.gis_version)) #d["has_ps"] = gis.Gis_points.select().where((gis.Gis_points.ptype == 'P') | (gis.Gis_points.ptype == 'I')).count() > 0 #d["has_res"] = gis.Gis_water.select().count() > 0 description = None conn = lib.open_db(project_db) if lib.exists_table(conn, 'object_cnt'): oc = Object_cnt.get_or_none() if oc is not None: description = oc.name d["project_description"] = description return d except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def put(self, project_db): parser = reqparse.RequestParser() parser.add_argument('name', type=str, required=True, location='json') parser.add_argument('description', type=str, required=True, location='json') args = parser.parse_args(strict=False) SetupProjectDatabase.init(project_db) try: m = Project_config.get() m.project_name = args['name'] result = m.save() oc = Object_cnt.get() oc.name = args['description'] result = oc.save() if result > 0: return 200 abort(400, message="Unable to update project configuration table.") except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.") except Object_cnt.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def __init__(self, project_db_file, delete_existing, create_stations, import_method='database', file1=None, file2=None): self.__abort = False SetupProjectDatabase.init(project_db_file) self.project_db_file = project_db_file self.project_db = project_base.db self.create_stations = create_stations self.import_method = import_method self.file1 = file1 self.file2 = file2 try: config = Project_config.get() if self.import_method == 'database' and self.project_db_file != '' and self.project_db_file is not None: wgn_db = utils.full_path(self.project_db_file, config.wgn_db) if not os.path.exists(wgn_db): sys.exit('WGN path {dir} does not exist.'.format(dir=wgn_db)) if config.wgn_table_name is None: sys.exit('Weather generator table name not set in config table.') self.wgn_database = wgn_db self.wgn_table = config.wgn_table_name except Project_config.DoesNotExist: sys.exit('Could not retrieve project configuration from database') if delete_existing: self.delete_existing()
def get(self, project_db): SetupProjectDatabase.init(project_db) try: c = Project_config.get() config = get_model_to_dict_dates(c, project_db) t = Time_sim.get_or_create_default() time = model_to_dict(t) m = Print_prt.get() prt = model_to_dict(m, recurse=False) o = Print_prt_object.select() objects = [model_to_dict(v, recurse=False) for v in o] prt = {'prt': prt, 'objects': objects} return { 'config': config, 'time': time, 'print': prt, 'imported_weather': climate.Weather_sta_cli.select().count() > 0 and climate.Weather_wgn_cli.select().count() > 0, 'has_observed_weather': climate.Weather_sta_cli.observed_count() > 0 } except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.") except Print_prt.DoesNotExist: abort(404, message="Could not retrieve print_prt data.")
def __init__(self, project_db_file): self.__abort = False SetupProjectDatabase.init(project_db_file) self.project_db = project_base.db try: config = Project_config.get() input_files_dir = utils.full_path(project_db_file, config.input_files_dir).replace( "\\", "/") if not os.path.exists(input_files_dir): sys.exit( 'The input files directory {dir} does not exist. Please select a valid path and try again.' .format(dir=input_files_dir)) weather_data_dir = None if config.weather_data_dir is not None: weather_data_dir = utils.full_path( project_db_file, config.weather_data_dir).replace("\\", "/") if not os.path.exists(weather_data_dir): sys.exit( 'Weather data directory {dir} does not exist.'.format( dir=weather_data_dir)) self.__dir = input_files_dir self.__weather_dir = weather_data_dir self.__version = config.editor_version self.__current_progress = 0 self.__is_lte = config.is_lte except Project_config.DoesNotExist: sys.exit('Could not retrieve project configuration from database')
def create(self): SetupProjectDatabase.create_tables() SetupProjectDatabase.initialize_data("demo") base_path = os.path.dirname(self.project_db) rel_project_db = os.path.relpath(self.project_db, base_path) rel_reference_db = os.path.relpath(self.reference_db, base_path) print("\t - project_db {}".format(self.project_db)) print("\t - base_path {}".format(base_path)) # print("rel_project_db {}".format(rel_project_db)) # print("rel_reference_db {}".format(rel_reference_db)) Project_config.get_or_create_default( editor_version=self.editor_version, project_name=self.project_name, #project_db=rel_project_db, reference_db=rel_reference_db)
def get(self, project_db): check_config(project_db) SetupProjectDatabase.init(project_db) try: m = Project_config.get() return get_model_to_dict_dates(m, project_db) except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def get(self, project_db): SetupProjectDatabase.init(project_db) try: m = Project_config.get() gis_type = 'QSWAT+ ' if m.gis_type == 'qgis' else 'GIS ' info = { 'name': m.project_name, 'is_lte': m.is_lte, 'status': { 'imported_weather': climate.Weather_sta_cli.select().count() > 0 and climate.Weather_wgn_cli.select().count() > 0, 'wrote_inputs': m.input_files_last_written is not None, 'ran_swat': m.swat_last_run is not None, 'imported_output': m.output_last_imported is not None }, 'simulation': model_to_dict(simulation.Time_sim.get_or_none()), 'total_area': gis.Gis_subbasins.select(fn.Sum( gis.Gis_subbasins.area)).scalar(), 'totals': { 'hru': connect.Hru_con.select().count(), 'lhru': connect.Hru_lte_con.select().count(), 'rtu': connect.Rout_unit_con.select().count(), 'mfl': connect.Modflow_con.select().count(), 'aqu': connect.Aquifer_con.select().count(), 'cha': connect.Channel_con.select().count(), 'res': connect.Reservoir_con.select().count(), 'rec': connect.Recall_con.select().count(), 'exco': connect.Exco_con.select().count(), 'dlr': connect.Delratio_con.select().count(), 'out': connect.Outlet_con.select().count(), 'lcha': connect.Chandeg_con.select().count(), 'aqu2d': connect.Aquifer2d_con.select().count(), 'lsus': regions.Ls_unit_def.select().count(), 'subs': gis.Gis_subbasins.select().count() }, 'editor_version': m.editor_version, 'gis_version': gis_type + m.gis_version } return info except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def put(self, project_db): SetupProjectDatabase.init(project_db) try: m = Project_config.get() m.output_last_imported = datetime.now() result = m.save() if result > 0: return 200 abort(400, message="Unable to update project configuration table.") except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def __init__(self, project_db_file, delete_existing, create_stations, source_dir): self.__abort = False SetupProjectDatabase.init(project_db_file) config = Project_config.get() weather_data_dir = utils.full_path(project_db_file, config.weather_data_dir) if not os.path.exists(weather_data_dir): sys.exit('Weather data directory {dir} does not exist.'.format(dir=weather_data_dir)) self.output_dir = weather_data_dir self.project_db_file = project_db_file self.project_db = project_base.db self.source_dir = source_dir self.delete_existing = delete_existing self.create_stations = create_stations
def import_data(self): try: config = Project_config.get() weather_data_dir = utils.full_path(self.project_db_file, config.weather_data_dir) if not os.path.exists(weather_data_dir): sys.exit('Weather data directory {dir} does not exist.'.format(dir=weather_data_dir)) self.add_weather_files(weather_data_dir) if self.create_stations: self.create_weather_stations(20, 45) self.match_stations(65) else: self.match_files_to_stations(20, 45) except Project_config.DoesNotExist: sys.exit('Could not retrieve project configuration from database')
def get(self, project_db): SetupProjectDatabase.init(project_db) try: m = Project_config.get() return { "has_weather": climate.Weather_sta_cli.select().count() > 0 and climate.Weather_wgn_cli.select().count() > 0, "input_files_dir": utils.full_path(project_db, m.input_files_dir), "input_files_last_written": utils.json_encode_datetime(m.input_files_last_written), "swat_last_run": utils.json_encode_datetime(m.swat_last_run) } except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def __init__(self, project_db, editor_version, swat_exe, weather_dir, weather_save_dir='', weather_import_format='plus', wgn_import_method='database', wgn_db='C:/SWAT/SWATPlus/Databases/swatplus_wgn.sqlite', wgn_table='wgn_cfsr_world', wgn_csv_sta_file=None, wgn_csv_mon_file=None, year_start=None, day_start=None, year_end=None, day_end=None, input_files_dir=None, swat_version=None): # Setup project databases and import GIS data SetupProject(project_db, editor_version, project_db.replace('.sqlite', '.json', 1)) rel_input_files = 'Scenarios/Default/TxtInOut' if input_files_dir is None else utils.rel_path( project_db, input_files_dir) input_files_path = utils.full_path(project_db, rel_input_files) if weather_import_format != 'plus' and weather_save_dir == '': weather_save_dir = rel_input_files # Set project config table arguments used by APIs m = Project_config.get() m.wgn_db = wgn_db m.wgn_table_name = wgn_table m.weather_data_dir = utils.rel_path( project_db, weather_dir ) if weather_import_format == 'plus' else utils.rel_path( project_db, weather_save_dir) m.input_files_dir = rel_input_files result = m.save() # Import WGN wgn_api = WgnImport(project_db, True, False, wgn_import_method, wgn_csv_sta_file, wgn_csv_mon_file) wgn_api.import_data() # Import weather files if weather_import_format == 'plus': weather_api = WeatherImport(project_db, True, True) weather_api.import_data() else: weather_api = Swat2012WeatherImport(project_db, True, True, weather_dir) weather_api.import_data() # Set time_sim if parameters given if year_start is not None: Time_sim.update_and_exec(day_start, year_start, day_end, year_end, 0) # Write input files write_api = WriteFiles(project_db, swat_version) write_api.write() # Run the model cwd = os.getcwd() os.chdir(input_files_path) run_result = os.system(swat_exe) print(run_result) # Import output files to db if successful run if run_result == 0: os.chdir(cwd) output_db_file = os.path.join(input_files_path, '../', 'Results', 'swatplus_output.sqlite') output_api = ReadOutput(input_files_path, output_db_file) output_api.read() m = Project_config.get() m.swat_last_run = datetime.now() m.output_last_imported = datetime.now() result = m.save()
def __init__(self, project_db, editor_version, project_name=None, datasets_db=None, constant_ps=True, is_lte=False, project_description=None): self.__abort = False base_path = os.path.dirname(project_db) rel_project_db = os.path.relpath(project_db, base_path) if datasets_db is None: conn = lib.open_db(project_db) if not lib.exists_table(conn, 'project_config'): sys.exit( 'No datasets database provided and the project_config table in your project database does not exist. Please provide either a datasets database file or an existing project database.' ) SetupProjectDatabase.init(project_db) try: config = Project_config.get() datasets_db = utils.full_path(project_db, config.reference_db) if project_name is None: project_name = config.project_name except Project_config.DoesNotExist: sys.exit('Could not retrieve project configuration data.') rel_datasets_db = os.path.relpath(datasets_db, base_path) ver_check = SetupDatasetsDatabase.check_version( datasets_db, editor_version) if ver_check is not None: sys.exit(ver_check) # Backup original db before beginning do_gis = False if os.path.exists(project_db): do_gis = True try: self.emit_progress(2, 'Backing up GIS database...') filename, file_extension = os.path.splitext(rel_project_db) bak_filename = filename + '_bak_' + time.strftime( '%Y%m%d-%H%M%S') + file_extension bak_dir = os.path.join(base_path, 'DatabaseBackups') if not os.path.exists(bak_dir): os.makedirs(bak_dir) backup_db_file = os.path.join(bak_dir, bak_filename) copyfile(project_db, backup_db_file) except IOError as err: sys.exit(err) try: SetupProjectDatabase.init(project_db, datasets_db) self.emit_progress(10, 'Creating database tables...') SetupProjectDatabase.create_tables() self.emit_progress(50, 'Copying data from SWAT+ datasets database...') description = project_description if project_description is not None else project_name SetupProjectDatabase.initialize_data( description, is_lte, overwrite_plants=OVERWRITE_PLANTS) config = Project_config.get_or_create_default( editor_version=editor_version, project_name=project_name, project_db=rel_project_db, reference_db=rel_datasets_db, project_directory='', is_lte=is_lte) conn = lib.open_db(project_db) plant_cols = lib.get_column_names(conn, 'plants_plt') plant_col_names = [v['name'] for v in plant_cols] if 'days_mat' not in plant_col_names: migrator = SqliteMigrator(SqliteDatabase(project_db)) migrate( migrator.rename_column('plants_plt', 'plnt_hu', 'days_mat')) for p in project_plants: dp = dataset_plants.get_or_none( dataset_plants.name == p.name) if dp is not None: p.days_mat = dp.days_mat else: p.days_mat = 0 p.save() except Exception as ex: if backup_db_file is not None: self.emit_progress(50, "Error occurred. Rolling back database...") SetupProjectDatabase.rollback(project_db, backup_db_file) self.emit_progress(100, "Error occurred.") sys.exit(str(ex)) if do_gis: api = GisImport(project_db, True, constant_ps, backup_db_file) api.insert_default()
def write(self): try: step = 3 small_step = 1 big_step = 5 bigger_step = 10 total = 0 self.write_simulation(total, step) total += step self.write_climate(total, bigger_step) total += bigger_step self.copy_weather_files(total, step) total += step self.write_connect(total, step) total += step self.write_channel(total, step) total += step self.write_reservoir(total, step) total += step self.write_routing_unit(total, step) total += step self.write_hru(total, bigger_step) total += bigger_step self.write_dr(total, small_step) total += small_step self.write_aquifer(total, small_step) total += small_step self.write_herd(total, small_step) total += small_step self.write_water_rights(total, small_step) total += small_step self.write_link(total, small_step) total += small_step self.write_basin(total, small_step) total += small_step self.write_hydrology(total, step) total += step self.write_exco(total, step) total += step self.write_recall(total, step) total += step self.write_structural(total, step) total += step self.write_parm_db(total, step) total += step self.write_ops(total, step) total += step self.write_lum(total, step) total += step self.write_chg(total, step) total += step self.write_init(total, step) total += step self.write_soils(total, bigger_step) total += bigger_step self.write_decision_table(total, step) total += step self.write_regions(total, step) total += step self.update_file_status(total, "file.cio") config.File_cio(os.path.join(self.__dir, "file.cio"), self.__version, self.__swat_version).write() Project_config.update(input_files_last_written=datetime.now(), swat_last_run=None, output_last_imported=None).execute() except ValueError as err: sys.exit(err)
def get_args(self, table_name, project_db, get_selected_ids=False, extra_args=[]): parser = reqparse.RequestParser() if get_selected_ids: parser.add_argument('selected_ids', type=int, action='append', required=False, location='json') else: parser.add_argument('id', type=int, required=False, location='json') parser.add_argument('name', type=str, required=False, location='json') parser.add_argument('description', type=str, required=False, location='json') for extra in extra_args: parser.add_argument(extra['name'], type=extra['type'], required=False, location='json') try: c = Project_config.get() datasets_db = c.reference_db if not os.path.exists(c.reference_db): datasets_db = os.path.normpath( os.path.join(os.path.dirname(project_db), c.reference_db)) SetupDatasetsDatabase.init(datasets_db) m = Var_range.select().where(Var_range.table == table_name) types = { 'float': float, 'int': int, 'text': str, 'string': str, 'select': str, 'lookup': str } for v in m: parser.add_argument(v.variable, type=types.get(v.type, str), required=False, location='json') except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.") args = parser.parse_args(strict=False) return args
def check_config(self, new_version): m = Project_config.get() if m.editor_version not in available_to_update: sys.exit("Unable to update this project to {new_version}. Updates from {current_version} unavailable.".format(new_version=new_version, current_version=m.editor_version)) return m
def get(self, project_db): SetupProjectDatabase.init(project_db) conn = lib.open_db(project_db) if not lib.exists_table(conn, 'chandeg_con'): abort(400, message='Project has not been set up.') try: m = Project_config.get() gis_type = 'QSWAT+ ' if m.gis_type == 'qgis' else 'GIS ' gis_text = '' if m.gis_version is None else gis_type + m.gis_version landuse_distrib = [] if m.gis_version is not None: landuse_distrib = gis.Gis_hrus.select( fn.Lower(gis.Gis_hrus.landuse).alias('name'), fn.Sum(gis.Gis_hrus.arslp).alias('y')).group_by( gis.Gis_hrus.landuse) current_path = os.path.dirname(project_db) scenarios_path = os.path.join(current_path, 'Scenarios') scenarios = [] if os.path.isdir(scenarios_path): for p in os.listdir(scenarios_path): if os.path.isdir(os.path.join( scenarios_path, p)) and p != 'Default' and p != 'default': db_files = [ f for f in os.listdir( os.path.join(scenarios_path, p)) if f.endswith('.sqlite') ] if len(db_files) > 0: scenarios.append({ 'name': p, 'path': os.path.join(scenarios_path, p, db_files[0]) }) oc = Object_cnt.get_or_none() info = { 'name': m.project_name, 'description': oc.name, 'file_path': current_path, 'last_modified': utils.json_encode_datetime( datetime.fromtimestamp(os.path.getmtime(project_db))), 'is_lte': m.is_lte, 'status': { 'imported_weather': climate.Weather_sta_cli.select().count() > 0 and climate.Weather_wgn_cli.select().count() > 0, 'wrote_inputs': m.input_files_last_written is not None, 'ran_swat': m.swat_last_run is not None, 'imported_output': m.output_last_imported is not None, 'using_gis': m.gis_version is not None }, 'simulation': model_to_dict(simulation.Time_sim.get_or_none()), 'total_area': connect.Rout_unit_con. select(fn.Sum(connect.Rout_unit_con.area)).scalar( ), #gis.Gis_subbasins.select(fn.Sum(gis.Gis_subbasins.area)).scalar(), 'totals': { 'hru': connect.Hru_con.select().count(), 'lhru': connect.Hru_lte_con.select().count(), 'rtu': connect.Rout_unit_con.select().count(), 'mfl': connect.Modflow_con.select().count(), 'aqu': connect.Aquifer_con.select().count(), 'cha': connect.Channel_con.select().count(), 'res': connect.Reservoir_con.select().count(), 'rec': connect.Recall_con.select().count(), 'exco': connect.Exco_con.select().count(), 'dlr': connect.Delratio_con.select().count(), 'out': connect.Outlet_con.select().count(), 'lcha': connect.Chandeg_con.select().count(), 'aqu2d': connect.Aquifer2d_con.select().count(), 'lsus': regions.Ls_unit_def.select().count(), 'subs': gis.Gis_subbasins.select().count() }, 'editor_version': m.editor_version, 'gis_version': gis_text, 'charts': { 'landuse': [{ 'name': o.name, 'y': o.y } for o in landuse_distrib] }, 'scenarios': scenarios } return info except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")
def put(self, project_db): parser = reqparse.RequestParser() parser.add_argument('input_files_dir', type=str, required=False, location='json') parser.add_argument('time', type=dict, required=False, location='json') parser.add_argument('print', type=dict, required=False, location='json') parser.add_argument('print_objects', type=list, required=False, location='json') args = parser.parse_args(strict=False) SetupProjectDatabase.init(project_db) try: m = Project_config.get() m.input_files_dir = utils.rel_path(project_db, args['input_files_dir']) result = m.save() time = args['time'] result = Time_sim.update_and_exec(time['day_start'], time['yrc_start'], time['day_end'], time['yrc_end'], time['step']) prt = args['print'] q = Print_prt.update(nyskip=prt['nyskip'], day_start=prt['day_start'], day_end=prt['day_end'], yrc_start=prt['yrc_start'], yrc_end=prt['yrc_end'], interval=prt['interval'], csvout=prt['csvout'], dbout=prt['dbout'], cdfout=prt['cdfout'], soilout=prt['soilout'], mgtout=prt['mgtout'], hydcon=prt['hydcon'], fdcout=prt['fdcout']) result = q.execute() prtObj = args['print_objects'] if prtObj is not None: for o in prtObj: Print_prt_object.update( daily=o['daily'], monthly=o['monthly'], yearly=o['yearly'], avann=o['avann']).where( Print_prt_object.id == o['id']).execute() if result > 0: return 200 abort(400, message="Unable to update project configuration table.") except Project_config.DoesNotExist: abort(404, message="Could not retrieve project configuration data.")