def main():
    
    ts = time.time()
    output_path = "../runs/phase_2/step" + str(step) + "/" + datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M')

    lookup_tables = read_lookup_table(lookup_table_file)

    s_res = ""
    c_res = ""

    for crop in crops:

        crop_id = ""
        if (crop == "maize"):
            crop_id = "SM"
        elif (crop == "winter_wheat"):
            crop_id = "WW"


        for res in resolutions:

            lookup_table = lookup_tables[res]
            #for k,v in lookup_table.iteritems():
            #    print k, v

            soil_file = ""
            if (step == 2) :

                s_res = "s" + str(res)
                c_res = "c1"

                soil_file = soil_files_directory + soil_files[res]
            elif (step == 3):

                s_res = "s1"
                c_res = "c" + str(res)

                soil_file = soil_files_directory + soil_files["1"]


            for simulation_type in simulation_types:
      
              print rank, "Simulation type: ", simulation_type, "\tResolution: ", res
              output_prefix, input_path, ini_file = getSimulationConfig(simulation_type, crop)
          
              splitted_meta_info_list = None
              
               
              if (rank == 0):
                  # only one processor reads in the meta information
                  meta_info_list = readMetaInfo(res, step, lookup_table)
                      
                  #for meta in meta_info_list:
                  #    print meta.start_date, meta.end_date, meta.project_id, meta.climate_file, meta.row_id, meta.col_id
                      
                  # split the meta info list into "number of processors" sublists 
                  splitted_meta_info_list = mpi_helper.splitListForNodes(meta_info_list, size)

              ###################################################
              # parallel part
              ##################################################
              
              # send each sublist of the splitted list to on processor
              node_specific_meta_info_list = comm.scatter(splitted_meta_info_list, root=0)      
          
              # each processor received a specific number of meta_info_objects
              # that he has to process
              print rank, "Received meta info list with", len(node_specific_meta_info_list), "elements" 

              start_date = str(start_year) + "-01-01"
              end_date = str(end_year) + "-01-31"

              monica_simulation_config = monica.MacsurScalingConfiguration()
              monica_simulation_config.setInputPath(input_path)
              monica_simulation_config.setIniFile(ini_file)
              monica_simulation_config.setCropName(crop)
              
              node_simulation_results = []
              
              for index, meta in enumerate(node_specific_meta_info_list):

                  #if (index >= 5):
                  #    continue

                  col_row_list = lookup_table[str([str(meta.col_id),str(meta.row_id)])]
                  lookup_col = int(col_row_list[0])
                  lookup_row = int(col_row_list[1])

                  lookup_project_id = "Res" + str(res) + "_C" + str(lookup_col) + ":R" + str(lookup_row)



                  print rank, "###################################"
                  print rank, str(index+1) +  "/" + str(len(node_specific_meta_info_list)) 
                  print rank, "Calculating", meta.project_id, "\tLookup_project_id:", lookup_project_id
                  print rank, meta.start_date, meta.end_date
                  print rank, meta.climate_file
                  print rank, soil_file

                  monica_simulation_config.setStartDate(meta.start_date)
                  monica_simulation_config.setEndDate(meta.end_date)
                  monica_simulation_config.setClimateFile(meta.climate_file)
                  monica_simulation_config.setProjectId(meta.project_id)
                  monica_simulation_config.setLookupProjectId(lookup_project_id)
                  monica_simulation_config.setRowId(meta.row_id)
                  monica_simulation_config.setColId(meta.col_id)
                  monica_simulation_config.setPhase(2)
                  monica_simulation_config.setStep(step)
                  monica_simulation_config.setSoilFile(soil_file)
                  
                  path = path = output_path + "/" + crop + "_" + simulation_type + "/" + meta.project_id
                  monica_simulation_config.setOutputPath(path) 
                  
                  if not (os.path.exists(path)):
                      print rank, "create_directory: ", path
                      os.makedirs(path)
                  
                  monica_simulation_config.setLatitude(meta.latitude)
                  monica_simulation_config.setElevation(meta.elevation)
                          
                  monica.activateDebugOutput(False);
                  monica.runMacsurScalingSimulation(monica_simulation_config)
                  sim_result = analyseSimulationOutput(path, meta.row_id, meta.col_id, crop)
                  node_simulation_results.extend(sim_result)
                  
                  # remove simulation result dir
                  if (remove_monica_files_after_simulation):
                      shutil.rmtree(path, True)

                  print rank, "###################################"
                  
                  #if (index == 1):
                  #    break
                  
          
              ###################################################
              # end of parallel part
              ##################################################              


                
              result_list = comm.gather(node_simulation_results, root=0)
            
              if (rank == 0):
                
                
                  output_filename = output_prefix + "_" + s_res + "x" + c_res + ".csv"       
                  output_filehandle = open(output_path + "/" + output_filename, "wb")
                  output_csv = csv.writer(output_filehandle, delimiter=sep)
                
                  header = ["gridcell", "year", "Yield (t DM/ha)", "Total above ground biomass (t DM/ha)",
                        "Total ET over the growing season (mm/growing season)", 
                        "Total intercepted PAR over the growing season (MJ/ha/growing season)",
                        "Maximum LAI during the growing season (m2/m2)", 
                        "Anthesis date (DOY)", "Maturity date (DOY)", 
                        "SOC at sowing date at 30 cm (gC/m2)", 
                        "SOC at sowing date at 2.0 m (gC/m2)", 
                        "Total net ecosystem exchange over the growing season (gC/m2/growing season)", 
                        "Total net primary productivity over the growing season (gC/m2/growing season)",
                        "Total N20 over the growing season (kg N/ha/growing season)",
                        "Total annual N20 (kg N/ha/year)",
                        "Total annual N leaching over 1.5 m (kg N/ha/year)", 
                        "Total N leaching over the growing season over 1.5 m (kg N/ha/growing season)",
                        "Total annual water loss below 1.5 m (mm/ha/year)", 
                        "Total water loss below 1.5 m over the growing season (mm/ha/growing season)", 
                        "Maximum rooted soil depth (m)",
                        "Total annual irrigated water amount (mm)"]
                  output_csv.writerow(header)
              
  
               
                  for node_list in result_list:   
                      for result_row in node_list:
                          output_csv.writerow(result_row)
                        
                  output_filehandle.close()
示例#2
0
def main():

    ts = time.time()
    output_path = "../runs/phase_2/step1/" + datetime.datetime.fromtimestamp(
        ts).strftime('%Y-%m-%d_%H-%M')

    for crop in crops:

        crop_id = ""
        if (crop == "maize"):
            crop_id = "SM"
        elif (crop == "winter_wheat"):
            crop_id = "WW"

        for c_res in climate_resolutions:

            for s_res in soil_resolutions:

                soil_file = soil_files_directory + soil_files[s_res]

                for simulation_type in simulation_types:

                    splitted_project_id_list = None
                    project_id_list = None

                    print rank, "Simulation type: ", simulation_type, "\Soil file: ", soil_file
                    output_prefix, input_path, ini_file = getSimulationConfig(
                        simulation_type, crop)

                    if (s_res == "sDS"):
                        print "Calculating the dominamt soil ..."
                        project_id_list = [[
                            "dominant_soil", 0, 0, 51.47, 175.5
                        ]]
                    else:
                        print "Calculating other different soil types ..."

                        if (rank == 0):
                            # only one processor reads in the meta information
                            project_id_list = readSoilFile(soil_file)

                    if (rank == 0):
                        # split the meta info list into "number of processors" sublists
                        splitted_project_id_list = mpi_helper.splitListForNodes(
                            project_id_list, size)

                    ###################################################
                    # parallel part
                    ##################################################

                    # send each sublist of the splitted list to on processor
                    node_specific_project_id_list = comm.scatter(
                        splitted_project_id_list, root=0)

                    # each processor received a specific number of meta_info_objects
                    # that he has to process
                    print rank, "Received project id list with", len(
                        node_specific_project_id_list), "elements"

                    start_date = str(start_year) + "-01-01"
                    end_date = str(end_year) + "-01-31"

                    monica_simulation_config = monica.MacsurScalingConfiguration(
                    )
                    monica_simulation_config.setInputPath(input_path)
                    monica_simulation_config.setIniFile(ini_file)
                    monica_simulation_config.setCropName(crop)
                    monica_simulation_config.setStartDate(start_date)
                    monica_simulation_config.setEndDate(end_date)
                    monica_simulation_config.setPhase(2)
                    monica_simulation_config.setStep(1)
                    monica_simulation_config.setSoilFile(soil_file)

                    node_simulation_results = []

                    for index, config_list in enumerate(
                            node_specific_project_id_list):
                        project_id = config_list[0]

                        row = config_list[1]
                        col = config_list[2]
                        latitude = config_list[3]
                        elevation = config_list[4]

                        print rank, "###################################"
                        print rank, str(index + 1) + "/" + str(
                            len(node_specific_project_id_list))
                        print rank, "Calculating", project_id
                        print rank, start_date, end_date
                        print rank, row, col, latitude, elevation

                        monica_simulation_config.setClimateFile(
                            "../input_data/phase_2/step1/climate_files/daily_mean_NRW.csv"
                        )

                        monica_simulation_config.setProjectId(project_id)
                        monica_simulation_config.setRowId(row)
                        monica_simulation_config.setColId(col)

                        path = output_path + "/" + crop + "_" + simulation_type + "/" + project_id
                        monica_simulation_config.setOutputPath(path)

                        if not (os.path.exists(path)):
                            print rank, "create_directory: ", path
                            os.makedirs(path)

                        monica_simulation_config.setLatitude(latitude)
                        monica_simulation_config.setElevation(elevation)

                        monica.activateDebugOutput(False)
                        monica.runMacsurScalingSimulation(
                            monica_simulation_config)
                        sim_result = analyseSimulationOutput(
                            path, row, col, crop)
                        node_simulation_results.extend(sim_result)

                        # remove simulation result dir
                        if (remove_monica_files_after_simulation):
                            shutil.rmtree(path, True)

                        print rank, "###################################"

                        #if (index == 1):
                        #    break

                    ###################################################
                    # end of parallel part
                    ##################################################

                    result_list = comm.gather(node_simulation_results, root=0)

                    if (rank == 0):

                        output_filename = output_prefix + "_" + s_res + "x" + c_res + ".csv"
                        output_filehandle = open(
                            output_path + "/" + output_filename, "wb")
                        output_csv = csv.writer(output_filehandle,
                                                delimiter=sep)

                        header = [
                            "gridcell", "year", "Yield (t DM/ha)",
                            "Total above ground biomass (t DM/ha)",
                            "Total ET over the growing season (mm/growing season)",
                            "Total intercepted PAR over the growing season (MJ/ha/growing season)",
                            "Maximum LAI during the growing season (m2/m2)",
                            "Anthesis date (DOY)", "Maturity date (DOY)",
                            "SOC at sowing date at 30 cm (gC/m2)",
                            "SOC at sowing date at 2.0 m (gC/m2)",
                            "Total net ecosystem exchange over the growing season (gC/m2/growing season)",
                            "Total net primary productivity over the growing season (gC/m2/growing season)",
                            "Total N20 over the growing season (kg N/ha/growing season)",
                            "Total annual N20 (kg N/ha/year)",
                            "Total annual N leaching over 1.5 m (kg N/ha/year)",
                            "Total N leaching over the growing season over 1.5 m (kg N/ha/growing season)",
                            "Total annual water loss below 1.5 m (mm/ha/year)",
                            "Total water loss below 1.5 m over the growing season (mm/ha/growing season)",
                            "Maximum rooted soil depth (m)",
                            "Total annual irrigated water amount (mm)"
                        ]
                        output_csv.writerow(header)

                        for node_list in result_list:
                            for result_row in node_list:
                                output_csv.writerow(result_row)

                        output_filehandle.close()
def main():
    
    ts = time.time()
    output_path = "../runs/phase_2/step1/" + datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M')

    for crop in crops:

        crop_id = ""
        if (crop == "maize"):
            crop_id = "SM"
        elif (crop == "winter_wheat"):
            crop_id = "WW"

        for c_res in climate_resolutions:

            for s_res in soil_resolutions:

                soil_file = soil_files_directory + soil_files[s_res]

                for simulation_type in simulation_types:
          
                    splitted_project_id_list = None    
                    project_id_list = None
              
                    print rank, "Simulation type: ", simulation_type, "\Soil file: ", soil_file
                    output_prefix, input_path, ini_file = getSimulationConfig(simulation_type, crop)
              
                    if (s_res == "sDS"):
                        print "Calculating the dominamt soil ..."
                        project_id_list = [["dominant_soil",0,0,51.47, 175.5]]
                    else:
                        print "Calculating other different soil types ..."
                   
                        if (rank == 0):
                            # only one processor reads in the meta information
                            project_id_list = readSoilFile(soil_file)
                          
                    if (rank == 0):
                        # split the meta info list into "number of processors" sublists 
                        splitted_project_id_list = mpi_helper.splitListForNodes(project_id_list, size)

                  
                    ###################################################
                    # parallel part
                    ##################################################
                  
                    # send each sublist of the splitted list to on processor
                    node_specific_project_id_list = comm.scatter(splitted_project_id_list, root=0)      
              
                    # each processor received a specific number of meta_info_objects
                    # that he has to process
                    print rank, "Received project id list with", len(node_specific_project_id_list), "elements" 

                    start_date = str(start_year) + "-01-01"
                    end_date = str(end_year) + "-01-31"

                    monica_simulation_config = monica.MacsurScalingConfiguration()
                    monica_simulation_config.setInputPath(input_path)
                    monica_simulation_config.setIniFile(ini_file)
                    monica_simulation_config.setCropName(crop)
                    monica_simulation_config.setStartDate(start_date)
                    monica_simulation_config.setEndDate(end_date)
                    monica_simulation_config.setPhase(2)
                    monica_simulation_config.setStep(1)
                    monica_simulation_config.setSoilFile(soil_file)
                  
                    node_simulation_results = []
                  
                    for index, config_list in enumerate(node_specific_project_id_list):
                        project_id = config_list[0]


                        row = config_list[1]
                        col = config_list[2]
                        latitude = config_list[3]
                        elevation = config_list[4]

                        print rank, "###################################"
                        print rank, str(index+1) +  "/" + str(len(node_specific_project_id_list)) 
                        print rank, "Calculating", project_id
                        print rank, start_date, end_date
                        print rank, row, col, latitude, elevation

                        monica_simulation_config.setClimateFile("../input_data/phase_2/step1/climate_files/daily_mean_NRW.csv")
              
                        monica_simulation_config.setProjectId(project_id)
                        monica_simulation_config.setRowId(row)
                        monica_simulation_config.setColId(col)
                        
                        path = output_path + "/" + crop + "_" + simulation_type + "/" + project_id
                        monica_simulation_config.setOutputPath(path) 
                            
                        if not (os.path.exists(path)):
                            print rank, "create_directory: ", path
                            os.makedirs(path)
                        
                        monica_simulation_config.setLatitude(latitude)
                        monica_simulation_config.setElevation(elevation)
                                
                        monica.activateDebugOutput(False);
                        monica.runMacsurScalingSimulation(monica_simulation_config)
                        sim_result = analyseSimulationOutput(path, row, col, crop)
                        node_simulation_results.extend(sim_result)
                        
                        # remove simulation result dir
                        if (remove_monica_files_after_simulation):
                            shutil.rmtree(path, True)
        
                        print rank, "###################################"
                        
                        #if (index == 1):
                        #    break
                        
                
                    ###################################################
                    # end of parallel part
                    ##################################################
                    
                    result_list = comm.gather(node_simulation_results, root=0)
                    
                    if (rank == 0):
                        
                        
                        output_filename = output_prefix + "_" + s_res + "x" + c_res + ".csv"       
                        output_filehandle = open(output_path + "/" + output_filename, "wb")
                        output_csv = csv.writer(output_filehandle, delimiter=sep)
                        
                        header = ["gridcell", "year", "Yield (t DM/ha)", "Total above ground biomass (t DM/ha)",
                                "Total ET over the growing season (mm/growing season)", 
                                "Total intercepted PAR over the growing season (MJ/ha/growing season)",
                                "Maximum LAI during the growing season (m2/m2)", 
                                "Anthesis date (DOY)", "Maturity date (DOY)", 
                                "SOC at sowing date at 30 cm (gC/m2)", 
                                "SOC at sowing date at 2.0 m (gC/m2)", 
                                "Total net ecosystem exchange over the growing season (gC/m2/growing season)", 
                                "Total net primary productivity over the growing season (gC/m2/growing season)",
                                "Total N20 over the growing season (kg N/ha/growing season)",
                                "Total annual N20 (kg N/ha/year)",
                                "Total annual N leaching over 1.5 m (kg N/ha/year)", 
                                "Total N leaching over the growing season over 1.5 m (kg N/ha/growing season)",
                                "Total annual water loss below 1.5 m (mm/ha/year)", 
                                "Total water loss below 1.5 m over the growing season (mm/ha/growing season)", 
                                "Maximum rooted soil depth (m)",
                                "Total annual irrigated water amount (mm)"]
                        output_csv.writerow(header)
                      
          
                       
                        for node_list in result_list:   
                            for result_row in node_list:
                                output_csv.writerow(result_row)
                                
                        output_filehandle.close()
示例#4
0
def main():

    ts = time.time()
    output_path = path_to_macsur_data + "/runs/phase_1/" + datetime.datetime.fromtimestamp(
        ts).strftime('%Y-%m-%d_%H-%M')

    for crop in macsur_config.crops:

        crop_id = ""
        if (crop == "maize"):
            crop_id = "SM"
        elif (crop == "winter_wheat"):
            crop_id = "WW"

        for simulation_type in macsur_config.simulation_types:

            for resolution in macsur_config.resolutions:

                print rank, "Simulation type: ", simulation_type, "\tResolution: ", resolution
                output_prefix, input_path, ini_file = getSimulationConfig(
                    simulation_type, crop)

                splitted_meta_info_list = None

                if (rank == 0):
                    # only one processor reads in the meta information
                    meta_info_list = readMetaInfo(resolution)

                    #for meta in meta_info_list:
                    #print meta.start_date, meta.end_date, meta.project_id, meta.climate_file, meta.row_id, meta.col_id

                    # split the meta info list into "number of processors" sublists
                    splitted_meta_info_list = mpi_helper.splitListForNodes(
                        meta_info_list, size)

                ###################################################
                # parallel part
                ##################################################

                # send each sublist of the splitted list to on processor
                node_specific_meta_info_list = comm.scatter(
                    splitted_meta_info_list, root=0)

                # each processor received a specific number of meta_info_objects
                # that he has to process
                print rank, "Received meta info list with", len(
                    node_specific_meta_info_list), "elements"

                monica_simulation_config = monica.MacsurScalingConfiguration()
                monica_simulation_config.setInputPath(input_path)
                monica_simulation_config.setIniFile(ini_file)
                monica_simulation_config.setCropName(crop)

                node_simulation_results = []

                for index, meta in enumerate(node_specific_meta_info_list):
                    print rank, "###################################"
                    print rank, str(index + 1) + "/" + str(
                        len(node_specific_meta_info_list))
                    print rank, "Calculating", meta.project_id
                    print rank, meta.start_date, meta.end_date
                    print rank, meta.climate_file
                    monica_simulation_config.setStartDate(meta.start_date)
                    monica_simulation_config.setEndDate(meta.end_date)
                    monica_simulation_config.setClimateFile(meta.climate_file)
                    monica_simulation_config.setProjectId(meta.project_id)
                    monica_simulation_config.setRowId(meta.row_id)
                    monica_simulation_config.setColId(meta.col_id)
                    monica_simulation_config.setPhase(1)

                    path = output_path + "/" + crop + "_" + simulation_type + "/res" + str(
                        meta.resolution) + "/C" + str(
                            meta.col_id) + "-R" + str(meta.row_id)
                    monica_simulation_config.setOutputPath(path)

                    if not (os.path.exists(path)):
                        print rank, "create_directory: ", path
                        os.makedirs(path)

                    monica_simulation_config.setLatitude(meta.latitude)
                    monica_simulation_config.setElevation(meta.elevation)

                    monica.activateDebugOutput(False)
                    monica.runMacsurScalingSimulation(monica_simulation_config)
                    sim_result = analyseSimulationOutput(
                        path, meta.row_id, meta.col_id, crop)
                    node_simulation_results.extend(sim_result)

                    # remove simulation result dir
                    if (remove_monica_files_after_simulation):
                        shutil.rmtree(path, True)

                    print rank, "###################################"

                    #if (index == 1):
                    #    break

                ###################################################
                # end of parallel part
                ##################################################

                result_list = comm.gather(node_simulation_results, root=0)

                if (rank == 0):

                    output_filename = output_prefix + "_Res" + str(
                        resolution) + "_" + crop + ".csv"
                    output_filehandle = open(
                        output_path + "/" + output_filename, "wb")
                    output_csv = csv.writer(output_filehandle, delimiter=sep)

                    header = [
                        "gridcell", "year", "Yield (t DM/ha)",
                        "Total above ground biomass (t DM/ha)",
                        "Total ET over the growing season (mm/growing season)",
                        "Total intercepted PAR over the growing season (MJ/ha/growing season)",
                        "Maximum LAI during the growing season (m2/m2)",
                        "Anthesis date (DOY)", "Maturity date (DOY)",
                        "SOC at sowing date at 30 cm (gC/m2)",
                        "SOC at sowing date at 2.0 m (gC/m2)",
                        "Total net ecosystem exchange over the growing season (gC/m2/growing season)",
                        "Total net primary productivity over the growing season (gC/m2/growing season)",
                        "Total N20 over the growing season (kg N/ha/growing season)",
                        "Total annual N20 (kg N/ha/year)",
                        "Total annual N leaching over 1.5 m (kg N/ha/year)",
                        "Total N leaching over the growing season over 1.5 m (kg N/ha/growing season)",
                        "Total annual water loss below 1.5 m (mm/ha/year)",
                        "Total water loss below 1.5 m over the growing season (mm/ha/growing season)",
                        "Maximum rooted soil depth (m)",
                        "Total annual irrigated water amount (mm)"
                    ]
                    output_csv.writerow(header)

                    for node_list in result_list:
                        for result_row in node_list:
                            output_csv.writerow(result_row)

                    output_filehandle.close()