def write_ecoflow_file_drainageareaxml(file_list, dir_name, file_name): """ Write a csv file containing a label (basin id number) and its corresponding area. Parameters ---------- file_list : list List of WATERSimulation.xml files to process dir_name : string String name for output directory file_name : string String name for output file """ area_data = {} for f in file_list: filedir, filename = helpers.get_file_info(f) ecoflow_dir = helpers.make_directory(path=filedir, directory_name=dir_name) helpers.print_input_output_info( input_dict={"input_file": f}, output_dict={"output_directory": ecoflow_dir}) waterapputils_logging.initialize_loggers(output_dir=ecoflow_dir) # read xml file waterxml_tree = waterxml.read_file(f) # get area from each region from the xml file and sum for a total area project, study, simulation = waterxml.get_xml_data( waterxml_tree=waterxml_tree) # get the project name which is the same as the stationid stationid = project["ProjName"] # get the area means for each region areas = waterxml.get_study_unit_areas(simulation_dict=simulation) # calculate total area total_area = waterxml.calc_total_study_unit_areas(areas) # fill area_data with total area area_data[stationid] = total_area # convert from km**2 to mi**2 area_data = helpers.convert_area_values(area_data, in_units="km2", out_units="mi2") # write timeseries of dishcarge + water use for ecoflow program watertxt.write_drainagearea_file(area_data=area_data, save_path=ecoflow_dir, filename=file_name) waterapputils_logging.remove_loggers()
def process_water_files(file_list, settings, print_data=True): """ Process a list of WATER xml files according to options contained in arguments parameter. Parameters ---------- file_list : list List of files to parse, process, and plot. arguments : argparse object An argparse object containing user options. """ print("Processing WATER files ...\n") for f in file_list: ext = os.path.splitext(f)[1] assert ext == ".txt" or ext == ".xml", "Can not process file {}. File extension {} is not .txt or .xml".format( f, ext) filedir, filename = helpers.get_file_info(f) if ext == ".txt": output_dir = helpers.make_directory( path=filedir, directory_name=settings["watertxt_directory_name"]) helpers.print_input_output_info( input_dict={"input_file": f}, output_dict={"output_directory": output_dir}) waterapputils_logging.initialize_loggers(output_dir=output_dir) data = watertxt.read_file(f) watertxt_viewer.plot_watertxt_data(data, save_path=output_dir) if print_data: watertxt_viewer.print_watertxt_data(data) elif ext == ".xml": output_dir = helpers.make_directory( path=filedir, directory_name=settings["waterxml_directory_name"]) waterapputils_logging.initialize_loggers(output_dir=output_dir) helpers.print_input_output_info( input_dict={"input_file": f}, output_dict={"output_directory": output_dir}) data = waterxml.read_file(f) waterxml_viewer.plot_waterxml_timeseries_data(data, save_path=output_dir) waterxml_viewer.plot_waterxml_topographic_wetness_index_data( data, save_path=output_dir) if print_data: waterxml_viewer.print_waterxml_data(data) waterapputils_logging.remove_loggers()
def write_ecoflow_file_drainageareaxml(file_list, dir_name, file_name): """ Write a csv file containing a label (basin id number) and its corresponding area. Parameters ---------- file_list : list List of WATERSimulation.xml files to process dir_name : string String name for output directory file_name : string String name for output file """ area_data = {} for f in file_list: filedir, filename = helpers.get_file_info(f) ecoflow_dir = helpers.make_directory(path = filedir, directory_name = dir_name) helpers.print_input_output_info(input_dict = {"input_file": f}, output_dict = {"output_directory": ecoflow_dir}) waterapputils_logging.initialize_loggers(output_dir = ecoflow_dir) # read xml file waterxml_tree = waterxml.read_file(f) # get area from each region from the xml file and sum for a total area project, study, simulation = waterxml.get_xml_data(waterxml_tree = waterxml_tree) # get the project name which is the same as the stationid stationid = project["ProjName"] # get the area means for each region areas = waterxml.get_study_unit_areas(simulation_dict = simulation) # calculate total area total_area = waterxml.calc_total_study_unit_areas(areas) # fill area_data with total area area_data[stationid] = total_area # convert from km**2 to mi**2 area_data = helpers.convert_area_values(area_data, in_units = "km2", out_units = "mi2") # write timeseries of dishcarge + water use for ecoflow program watertxt.write_drainagearea_file(area_data = area_data, save_path = ecoflow_dir, filename = file_name) waterapputils_logging.remove_loggers()
def process_water_files(file_list, settings, print_data=True): """ Process a list of WATER xml files according to options contained in arguments parameter. Parameters ---------- file_list : list List of files to parse, process, and plot. arguments : argparse object An argparse object containing user options. """ print("Processing WATER files ...\n") for f in file_list: ext = os.path.splitext(f)[1] assert ext == ".txt" or ext == ".xml", "Can not process file {}. File extension {} is not .txt or .xml".format( f, ext ) filedir, filename = helpers.get_file_info(f) if ext == ".txt": output_dir = helpers.make_directory(path=filedir, directory_name=settings["watertxt_directory_name"]) helpers.print_input_output_info(input_dict={"input_file": f}, output_dict={"output_directory": output_dir}) waterapputils_logging.initialize_loggers(output_dir=output_dir) data = watertxt.read_file(f) watertxt_viewer.plot_watertxt_data(data, save_path=output_dir) if print_data: watertxt_viewer.print_watertxt_data(data) elif ext == ".xml": output_dir = helpers.make_directory(path=filedir, directory_name=settings["waterxml_directory_name"]) waterapputils_logging.initialize_loggers(output_dir=output_dir) helpers.print_input_output_info(input_dict={"input_file": f}, output_dict={"output_directory": output_dir}) data = waterxml.read_file(f) waterxml_viewer.plot_waterxml_timeseries_data(data, save_path=output_dir) waterxml_viewer.plot_waterxml_topographic_wetness_index_data(data, save_path=output_dir) if print_data: waterxml_viewer.print_waterxml_data(data) waterapputils_logging.remove_loggers()
def process_cmp(file_list, settings, print_data=True): """ Compare two WATER text files according to options contained in arguments parameter. Parameters ---------- file_list : list List of files to parse, process, and plot. arguments : argparse object An argparse object containing user options. """ print("Comparing WATER files ...\n") water_file1 = file_list[0] water_file2 = file_list[1] filedir1, filename1 = helpers.get_file_info(water_file1) filedir2, filename2 = helpers.get_file_info(water_file2) ext1 = os.path.splitext(filename1)[1] ext2 = os.path.splitext(filename2)[1] assert ext1 == ".txt" or ext1 == ".xml", "Can not process file {}. File extension {} is not .txt or .xml".format( filename1, ext1) assert ext2 == ".txt" or ext2 == ".xml", "Can not process file {}. File extension {} is not .txt or .xml".format( filename2, ext2) if ext1 == ".txt" and ext2 == ".txt": output_dir = helpers.make_directory( path=filedir1, directory_name=settings["watertxt_directory_name"]) helpers.print_input_output_info( input_dict={ "input_file_1": water_file1, "input_file_2": water_file2 }, output_dict={"output_directory": output_dir}) waterapputils_logging.initialize_loggers(output_dir=output_dir) watertxt_data1 = watertxt.read_file(water_file1) watertxt_data2 = watertxt.read_file(water_file2) watertxt_viewer.plot_watertxt_comparison(watertxt_data1, watertxt_data2, save_path=output_dir) if print_data: watertxt_viewer.print_watertxt_data(watertxt_data1) watertxt_viewer.print_watertxt_data(watertxt_data2) elif ext1 == ".xml" and ext2 == ".xml": output_dir = helpers.make_directory( path=filedir1, directory_name=settings["waterxml_directory_name"]) helpers.print_input_output_info( input_dict={ "input_file_1": water_file1, "input_file_2": water_file2 }, output_dict={"output_directory": output_dir}) waterapputils_logging.initialize_loggers(output_dir=output_dir) waterxml_data1 = waterxml.read_file(water_file1) waterxml_data2 = waterxml.read_file(water_file2) waterxml_viewer.plot_waterxml_timeseries_comparison( waterxml_data1, waterxml_data2, save_path=output_dir) if print_data: waterxml_viewer.print_waterxml_data(waterxml_data1) waterxml_viewer.print_waterxml_data(waterxml_data2) else: print( "Can not process files {} and {}. File extensions {} and {} both need to be .txt or .xml" .format(filename1, filename2, ext1, ext2)) waterapputils_logging.remove_loggers()
def process_intersecting_tiles(intersecting_tiles, settings, gcm_delta_dir): """ Apply water use data to a WATER \*.txt file. The new file created is saved to the same directory as the \*.xml file. Parameters ---------- intersecting_tiles : dictionary Dictionary containing lists of values for a particular field that were intersected by another shapefile. settings : dictionary Dictionary of user settings gcm_delta_dir : string string path to ecoflow directory Notes ----- Uses settings set in user_settings.py """ # create a file for the output for featureid, tiles in intersecting_tiles.iteritems(): # get monthly average gcm delta values deltas_data_list, deltas_avg_dict = deltas.get_deltas(delta_files = settings["gcm_delta_files"], tiles = tiles) # print monthly output in nice format to info file print("FeatureId: {}\n Tiles: {}\n Average GCM Deltas:\n".format(featureid, tiles)) for key in deltas_avg_dict.keys(): print(" {}\n".format(key)) helpers.print_monthly_dict(monthly_dict = deltas_avg_dict[key]) # get the txt data file that has a parent directory matching the current featureid if settings["is_batch_simulation"]: path = os.path.join(settings["simulation_directory"], featureid) else: path = settings["simulation_directory"] # find the WATERSimulation.xml and WATER.txt files waterxml_file = helpers.find_file(name = settings["water_database_file_name"], path = path) watertxt_file = helpers.find_file(name = settings["water_text_file_name"], path = path) # get file info waterxml_dir, waterxml_filename = helpers.get_file_info(waterxml_file) watertxt_dir, watertxt_filename = helpers.get_file_info(watertxt_file) # create an output directory output_dir = helpers.make_directory(path = waterxml_dir, directory_name = settings["gcm_delta_directory_name"]) # initialize error logging waterapputils_logging.initialize_loggers(output_dir = output_dir) # read the xml file waterxml_tree = waterxml.read_file(waterxml_file) watertxt_data = watertxt.read_file(watertxt_file) # apply gcm delta for key, value in deltas_avg_dict.iteritems(): if key == "Ppt": waterxml.apply_factors(waterxml_tree = waterxml_tree, element = "ClimaticPrecipitationSeries", factors = deltas_avg_dict[key]) elif key == "Tmax": waterxml.apply_factors(waterxml_tree = waterxml_tree, element = "ClimaticTemperatureSeries", factors = deltas_avg_dict[key]) elif key == "PET": watertxt.apply_factors(watertxt_data, name = "PET", factors = deltas_avg_dict[key], is_additive = False) # update the project name in the updated xml project = waterxml.create_project_dict() project = waterxml.fill_dict(waterxml_tree = waterxml_tree, data_dict = project, element = "Project", keys = project.keys()) waterxml.change_element_value(waterxml_tree = waterxml_tree, element = "Project", child = "ProjName" , new_value = settings["gcm_delta_prepend_name"] + project["ProjName"]) # write updated xml waterxml_with_gcm_delta_file = settings["gcm_delta_prepend_name"] + waterxml_filename waterxml.write_file(waterxml_tree = waterxml_tree, save_path = output_dir, filename = waterxml_with_gcm_delta_file) # write the pet timeseries file watertxt.write_timeseries_file(watertxt_data, name = "PET", save_path = output_dir, filename = settings["pet_timeseries_file_name"]) # plot updated_waterxml_file = os.path.join(output_dir, waterxml_with_gcm_delta_file) water_files_processing.process_water_files(file_list = [updated_waterxml_file ], settings = settings, print_data = False) water_files_processing.process_cmp(file_list = [updated_waterxml_file, waterxml_file], settings = settings, print_data = False) # plot the gcm deltas for deltas_data in deltas_data_list: deltas_viewer.plot_deltas_data(deltas_data = deltas_data, save_path = helpers.make_directory(path = gcm_delta_dir, directory_name = settings["gcm_delta_directory_name"]))
def process_intersecting_tiles(intersecting_tiles, settings, gcm_delta_dir): """ Apply water use data to a WATER \*.txt file. The new file created is saved to the same directory as the \*.xml file. Parameters ---------- intersecting_tiles : dictionary Dictionary containing lists of values for a particular field that were intersected by another shapefile. settings : dictionary Dictionary of user settings gcm_delta_dir : string string path to ecoflow directory Notes ----- Uses settings set in user_settings.py """ # create a file for the output for featureid, tiles in intersecting_tiles.iteritems(): # get monthly average gcm delta values deltas_data_list, deltas_avg_dict = deltas.get_deltas( delta_files=settings["gcm_delta_files"], tiles=tiles) # print monthly output in nice format to info file print("FeatureId: {}\n Tiles: {}\n Average GCM Deltas:\n".format( featureid, tiles)) for key in deltas_avg_dict.keys(): print(" {}\n".format(key)) helpers.print_monthly_dict(monthly_dict=deltas_avg_dict[key]) # get the txt data file that has a parent directory matching the current featureid if settings["is_batch_simulation"]: path = os.path.join(settings["simulation_directory"], featureid) else: path = settings["simulation_directory"] # find the WATERSimulation.xml and WATER.txt files waterxml_file = helpers.find_file( name=settings["water_database_file_name"], path=path) watertxt_file = helpers.find_file( name=settings["water_text_file_name"], path=path) # get file info waterxml_dir, waterxml_filename = helpers.get_file_info(waterxml_file) watertxt_dir, watertxt_filename = helpers.get_file_info(watertxt_file) # create an output directory output_dir = helpers.make_directory( path=waterxml_dir, directory_name=settings["gcm_delta_directory_name"]) # initialize error logging waterapputils_logging.initialize_loggers(output_dir=output_dir) # read the xml file waterxml_tree = waterxml.read_file(waterxml_file) watertxt_data = watertxt.read_file(watertxt_file) # apply gcm delta for key, value in deltas_avg_dict.iteritems(): if key == "Ppt": waterxml.apply_factors(waterxml_tree=waterxml_tree, element="ClimaticPrecipitationSeries", factors=deltas_avg_dict[key]) elif key == "Tmax": waterxml.apply_factors(waterxml_tree=waterxml_tree, element="ClimaticTemperatureSeries", factors=deltas_avg_dict[key]) elif key == "PET": watertxt.apply_factors(watertxt_data, name="PET", factors=deltas_avg_dict[key], is_additive=False) # update the project name in the updated xml project = waterxml.create_project_dict() project = waterxml.fill_dict(waterxml_tree=waterxml_tree, data_dict=project, element="Project", keys=project.keys()) waterxml.change_element_value( waterxml_tree=waterxml_tree, element="Project", child="ProjName", new_value=settings["gcm_delta_prepend_name"] + project["ProjName"]) # write updated xml waterxml_with_gcm_delta_file = settings[ "gcm_delta_prepend_name"] + waterxml_filename waterxml.write_file(waterxml_tree=waterxml_tree, save_path=output_dir, filename=waterxml_with_gcm_delta_file) # write the pet timeseries file watertxt.write_timeseries_file( watertxt_data, name="PET", save_path=output_dir, filename=settings["pet_timeseries_file_name"]) # plot updated_waterxml_file = os.path.join(output_dir, waterxml_with_gcm_delta_file) water_files_processing.process_water_files( file_list=[updated_waterxml_file], settings=settings, print_data=False) water_files_processing.process_cmp( file_list=[updated_waterxml_file, waterxml_file], settings=settings, print_data=False) # plot the gcm deltas for deltas_data in deltas_data_list: deltas_viewer.plot_deltas_data( deltas_data=deltas_data, save_path=helpers.make_directory( path=gcm_delta_dir, directory_name=settings["gcm_delta_directory_name"]))
def _create_test_data2(): """ Create test data to use with tests """ fixture = {} fixture["data_file"] = \ """ <Project> <ProjID>1</ProjID> <UserName>jlant</UserName> <DateCreated>2014-04-22T10:00:00.0000-00:00</DateCreated> <ProjName>my-project2</ProjName> <Study> <StudyID>1</StudyID> <ProjID>1</ProjID> <StudyLocDecDeg>40.5, -75.9</StudyLocDecDeg> <StudyXLocation>1600000.0</StudyXLocation> <StudyYLocation>2100000.0</StudyYLocation> <StudyDescription>Test simulation</StudyDescription> <IsPointApproved>true</IsPointApproved> <IsDelineated>true</IsDelineated> <IsStudyApproved>true</IsStudyApproved> <StudySimulation> <SimulID>1</SimulID> <StudyID>1</StudyID> <RegionType>4</RegionType> <isInitialized>true</isInitialized> <isLoaded>true</isLoaded> <isCompleted>false</isCompleted> <SimulationFeatures> <AttID>1</AttID> <SimulID>1</SimulID> <AttName>Study Unit Total Area</AttName> <AttCode>1</AttCode> <AttMeanVal>100.0</AttMeanVal> <AttMinVal>90.0</AttMinVal> <AttMaxVal>110.0</AttMaxVal> <AttstdDev>0</AttstdDev> <AttDescription> Study unit total area</AttDescription> <AttUnitsCode>303</AttUnitsCode> <AttUnits>(sq Km)</AttUnits> </SimulationFeatures> <SimulationFeatures> <AttID>2</AttID> <SimulID>1</SimulID> <AttName>Total Estimated Stream Area</AttName> <AttCode>37</AttCode> <AttMeanVal>5</AttMeanVal> <AttMinVal>4</AttMinVal> <AttMaxVal>6</AttMaxVal> <AttstdDev>0</AttstdDev> <AttDescription>Estimated area of stream coverage</AttDescription> <AttUnitsCode>303</AttUnitsCode> <AttUnits>(sq Km)</AttUnits> </SimulationFeatures> <SimulationTopographicWetnessIndex> <BinID>1</BinID> <SimulID>1</SimulID> <BinValueMean>3.1</BinValueMean> <BinValueFraction>0.002</BinValueFraction> </SimulationTopographicWetnessIndex> <SimulationTopographicWetnessIndex> <BinID>2</BinID> <SimulID>1</SimulID> <BinValueMean>4.2</BinValueMean> <BinValueFraction>0.005</BinValueFraction> </SimulationTopographicWetnessIndex> <StudyUnitDischargeSeries> <SeriesID>1</SeriesID> <SimulID>1</SimulID> <SeriesDate>2014-01-01T00:00:00-05:00</SeriesDate> <SeriesValue>200.0</SeriesValue> <SeriesUnitsCode>54</SeriesUnitsCode> <SeriesUnit>mm per day</SeriesUnit> </StudyUnitDischargeSeries> <StudyUnitDischargeSeries> <SeriesID>2</SeriesID> <SimulID>1</SimulID> <SeriesDate>2014-01-02T00:00:00-05:00</SeriesDate> <SeriesValue>210.0</SeriesValue> <SeriesUnitsCode>54</SeriesUnitsCode> <SeriesUnit>mm per day</SeriesUnit> </StudyUnitDischargeSeries> <ClimaticPrecipitationSeries> <SeriesID>1</SeriesID> <SimulID>1</SimulID> <SeriesDate>2014-01-01T00:00:00-05:00</SeriesDate> <SeriesValue>6.0</SeriesValue> <SeriesUnitsCode>4</SeriesUnitsCode> <SeriesUnit>mm</SeriesUnit> </ClimaticPrecipitationSeries> <ClimaticPrecipitationSeries> <SeriesID>2</SeriesID> <SimulID>1</SimulID> <SeriesDate>2014-01-02T00:00:00-05:00</SeriesDate> <SeriesValue>9</SeriesValue> <SeriesUnitsCode>4</SeriesUnitsCode> <SeriesUnit>mm</SeriesUnit> </ClimaticPrecipitationSeries> <ClimaticTemperatureSeries> <SeriesID>1</SeriesID> <SimulID>1</SimulID> <SeriesDate>2014-01-01T00:00:00-05:00</SeriesDate> <SeriesValue>22.2</SeriesValue> <SeriesUnitsCode>31</SeriesUnitsCode> <SeriesUnit>Celsius</SeriesUnit> </ClimaticTemperatureSeries> <ClimaticTemperatureSeries> <SeriesID>2</SeriesID> <SimulID>1</SimulID> <SeriesDate>2014-01-02T00:00:00-05:00</SeriesDate> <SeriesValue>24.4</SeriesValue> <SeriesUnitsCode>31</SeriesUnitsCode> <SeriesUnit>Celsius</SeriesUnit> </ClimaticTemperatureSeries> </StudySimulation> </Study> </Project> """ fileobj = StringIO(fixture["data_file"]) xml_tree = waterxml.read_file(fileobj) return xml_tree
def process_cmp(file_list, settings, print_data=True): """ Compare two WATER text files according to options contained in arguments parameter. Parameters ---------- file_list : list List of files to parse, process, and plot. arguments : argparse object An argparse object containing user options. """ print("Comparing WATER files ...\n") water_file1 = file_list[0] water_file2 = file_list[1] filedir1, filename1 = helpers.get_file_info(water_file1) filedir2, filename2 = helpers.get_file_info(water_file2) ext1 = os.path.splitext(filename1)[1] ext2 = os.path.splitext(filename2)[1] assert ext1 == ".txt" or ext1 == ".xml", "Can not process file {}. File extension {} is not .txt or .xml".format( filename1, ext1 ) assert ext2 == ".txt" or ext2 == ".xml", "Can not process file {}. File extension {} is not .txt or .xml".format( filename2, ext2 ) if ext1 == ".txt" and ext2 == ".txt": output_dir = helpers.make_directory(path=filedir1, directory_name=settings["watertxt_directory_name"]) helpers.print_input_output_info( input_dict={"input_file_1": water_file1, "input_file_2": water_file2}, output_dict={"output_directory": output_dir}, ) waterapputils_logging.initialize_loggers(output_dir=output_dir) watertxt_data1 = watertxt.read_file(water_file1) watertxt_data2 = watertxt.read_file(water_file2) watertxt_viewer.plot_watertxt_comparison(watertxt_data1, watertxt_data2, save_path=output_dir) if print_data: watertxt_viewer.print_watertxt_data(watertxt_data1) watertxt_viewer.print_watertxt_data(watertxt_data2) elif ext1 == ".xml" and ext2 == ".xml": output_dir = helpers.make_directory(path=filedir1, directory_name=settings["waterxml_directory_name"]) helpers.print_input_output_info( input_dict={"input_file_1": water_file1, "input_file_2": water_file2}, output_dict={"output_directory": output_dir}, ) waterapputils_logging.initialize_loggers(output_dir=output_dir) waterxml_data1 = waterxml.read_file(water_file1) waterxml_data2 = waterxml.read_file(water_file2) waterxml_viewer.plot_waterxml_timeseries_comparison(waterxml_data1, waterxml_data2, save_path=output_dir) if print_data: waterxml_viewer.print_waterxml_data(waterxml_data1) waterxml_viewer.print_waterxml_data(waterxml_data2) else: print( "Can not process files {} and {}. File extensions {} and {} both need to be .txt or .xml".format( filename1, filename2, ext1, ext2 ) ) waterapputils_logging.remove_loggers()