def submit_request(shapefile, URI, d, timeStart, timeEnd, attribute, values, ofp, mode='individually'): ''' calls pyGDP.submitFeatureWeightedGridStatistics() either with single dataset, or list of datasets (depending on mode) resubmits after waiting in the case of a server error ''' if mode == 'together': outfile = os.path.join(outpath, os.path.split(URI)[1] + '.csv') record = ',all datasets written to {}\n'.format(outfile) else: outfile = os.path.join(outpath, d+'_'+URI[-5:]+'.csv') record = ',{}\n'.format(d) done = False # while loop to wait and then resubmit if server/network fails # for some reason this appears to work after an exception (the files keep coming in with the right names), but printing to the output screen stops while not done: try: cida_handle = pyGDP.submitFeatureWeightedGridStatistics(shapefile, URI, d, timeStart, timeEnd, attribute, values) done = True except Exception, e: print e print traceback.format_exc() print "waiting {} min before trying again...".format(restart_submit_after) time.sleep(restart_submit_after * 60) # wait n seconds before attempting to submit again print "resubmitting..." else: ofp.write(cida_handle + record) shutil.move(cida_handle, outfile) print "saved as: {}".format(outfile)
def submit_request(shapefile, URI, d, timeStart, timeEnd, attribute, values, ofp, mode='individually'): ''' calls pyGDP.submitFeatureWeightedGridStatistics() either with single dataset, or list of datasets (depending on mode) resubmits after waiting in the case of a server error ''' if mode == 'together': outfile = os.path.join(outpath, os.path.split(URI)[1] + '.csv') record = ',all datasets written to {}\n'.format(outfile) else: outfile = os.path.join(outpath, d + '_' + URI[-5:] + '.csv') record = ',{}\n'.format(d) done = False # while loop to wait and then resubmit if server/network fails # for some reason this appears to work after an exception (the files keep coming in with the right names), but printing to the output screen stops while not done: try: cida_handle = pyGDP.submitFeatureWeightedGridStatistics( shapefile, URI, d, timeStart, timeEnd, attribute, values) done = True except Exception, e: print e print traceback.format_exc() print "waiting {} min before trying again...".format( restart_submit_after) time.sleep(restart_submit_after * 60) # wait n seconds before attempting to submit again print "resubmitting..." else: ofp.write(cida_handle + record) shutil.move(cida_handle, outfile) print "saved as: {}".format(outfile)
for v in values: print v # Choose Colorado value = ['Colorado'] # Search for datasets dataSetURIs = pyGDP.getDataSetURI(anyText='prism') for dataset in dataSetURIs: print dataset # Set our datasetURI to the OPeNDAP/dods response for the prism dataset. dataSetURI = 'dods://cida.usgs.gov/thredds/dodsC/prism' # Get the available data types associated with the dataset dataTypes = pyGDP.getDataType(dataSetURI) for dataType in dataTypes: print dataType dataType = 'ppt' # Get available time range for the dataset. timeRange = pyGDP.getTimeRange(dataSetURI, dataType) for t in timeRange: print t # Execute a GeatureWeightedGridStatistics request and return the path to the output file. # Note that this is for one time step but could be for multiple. outputfile = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSetURI, dataType, timeRange[0], timeRange[0], usr_attribute, value, verbose=True) print outputfile
# Grab the values from 'area_name' and 'sample:CSC_Boundaries' usr_attribute = 'area_name' values = pyGDP.getValues(shapefile, usr_attribute) for v in values: print(v) usr_value = 'Southwest' dataSetURI = 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml' dataTypes = pyGDP.getDataType(dataSetURI) for d in dataTypes: print(d) dataType = 'Prcp' # Get available time range on the dataset timeRange = pyGDP.getTimeRange(dataSetURI, dataType) for t in timeRange: print(t) timeBegin = '1960-01-01T00:00:00.000Z' timeEnd = '1960-01-21T00:00:00.000Z' outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSetURI, dataType, timeBegin, timeEnd, usr_attribute, usr_value, gmlIDs=None, verbose=True)
values = pyGDP.getValues(shapefile, polyAttribute) for v in values: print v polyValue = 'BD_Mtn' dataSetURI = 'http://cida.usgs.gov/thredds/dodsC/loca_historical' datatypes = pyGDP.getDataType(dataSetURI) outputpath_earlys=list() DataTypes_list = ['pr_ACCESS1-0','pr_CanESM2', 'pr_CCSM4', 'pr_CESM1-BGC', 'pr_CMCC-CMS',\ 'pr_CNRM-CM5', 'pr_GFDL-CM3', 'pr_HadGEM2-CC', \ 'pr_HadGEM2-ES', 'pr_MIROC-ESM-CHEM', 'pr_MIROC-ESM', \ 'pr_CSIRO-Mk3-6-0', 'pr_GFDL-ESM2M'] for Type_item in DataTypes_list: for datatype in datatypes: if Type_item in datatype: outName = 'Loca_Historical_' + datatype + '.csv' if os.path.isfile(outName): outputpath_earlys.append(outputPath) else: outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSetURI, datatype, '1950-01-01', '2005-12-31', polyAttribute, polyValue, outputfname = outName) print(outputPath)
def main_func(curdir, set, region): def Region_lookup(region): region_properties = [] if region == 'nhru': region_properties.append(371) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d35da1e4b0ca184833940c') region_properties.append('false') if region == 'R01': region_properties.append(2462) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5244735ae4b05b217bada04e') region_properties.append('false') if region == 'R02': region_properties.append(4827) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52696784e4b0584cbe9168ee') region_properties.append('false') if region == 'R03': region_properties.append(9899) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5283bd23e4b047efbbb57922') region_properties.append('false') if region == 'R04': region_properties.append(5936) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5284ff57e4b063f258e61b9d') region_properties.append('false') if region == 'R05': region_properties.append(7182) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/528516bbe4b063f258e62161') region_properties.append('false') if region == 'R06': region_properties.append(2303) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d75296e4b055e0afd5be2c') region_properties.append('true') if region == 'R07': region_properties.append(8205) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52851cd5e4b063f258e643dd') region_properties.append('true') if region == 'R08': region_properties.append(4449) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52854695e4b063f258e6513c') region_properties.append('true') if region == 'R10L': region_properties.append(8603) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/520031dee4b0ad2d97189db2') region_properties.append('true') elif region =='R10U': region_properties.append(10299) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5216849ee4b0b45d6ba61e2e') region_properties.append('false') elif region == 'R11': region_properties.append(7373) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d1f9ebe4b08b18a62d586b') region_properties.append('true') elif region == 'R12': region_properties.append(7815) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5201328ae4b009d47a4c247a') region_properties.append('false') elif region == 'R13': region_properties.append(1958) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d752b9e4b055e0afd5be36') region_properties.append('false') elif region == 'R14': region_properties.append(3879) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52029c68e4b0e21cafa4b40c') region_properties.append('false') elif region == 'R15': region_properties.append(3441) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5285389ae4b063f258e64863') region_properties.append('false') elif region == 'R16': region_properties.append(2664) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52853f97e4b063f258e64875') region_properties.append('false') return region_properties def list_define(set): if set == 'cccma_cgmc3_1': return ['cccma_cgmc3'+str(scenario), 'http://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] elif set == 'cnrm_cm3': return ['cnrm_cm3'+str(scenario), 'http://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] elif set == 'gfdl_cm2_1': return ['gfdl_cm2_1'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] elif set == 'ipsl_cm4': return ['ipsl_cm4'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] elif set == 'miroc3_2_medres': return ['miroc3_2_medres'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] elif set == 'miub_echo_g': return ['miub_echo_g'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] elif set == 'mri_cgcm2_3_2a': return ['mri_cgcm2_3_2a'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] elif set == 'mpi_echam5': return ['mpi_echam5'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] global csvfile, vt_dataset, nhru, length, vt_datatype, url import pyGDP Region_return=Region_lookup(region) hrus = Region_return[0] nhru = hrus ScienceBase_URL= Region_return[1] pyGDP.WFS_URL = ScienceBase_URL url = pyGDP.WFS_URL pyGDP = pyGDP.pyGDPwebProcessing() # change working directory so the GDP output will be written there # Datasets and their properties # run 1, 2 , and 3 #********************************** # run 1 only #scenario = 'a2' # other scenarios are 'a1b' and 'b1' #scenarios = ['a2','a1b','b1'] scenarios = ['a2', 'a1b'] timestart = '2046-01-15T00:00:00.000Z' timeend='2100-12-15T00:00:00.000Z' #********************************** # Datasets for each scenario - note that mpi_echam5 is not run for scenario a1b #a1b #data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mri_cgcm2_3_2a] #a2 #data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mpi_echam5,mri_cgcm2_3_2a] #b1 #data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mpi_echam5,mri_cgcm2_3_2a] # get list of shapefiles uploaded to the GDP shapefiles = pyGDP.getShapefiles() for shp in shapefiles: print shp # feature loaded from sciencebase #should shapefile be sb:SP_hru instead? shapefile = 'sb:nhru' user_attribute = 'hru_id_loc' user_value = None os.chdir(curdir) dir = os.getcwd() vt_data = list_define(set) vt_datatype = vt_data[5:] #for scenario in scenarios: # cnrm_cm3 = ['cnrm_cm3'+str(scenario), 'http://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ # 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', # 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', # 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] # # gfdl_cm2_1 = ['gfdl_cm2_1'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ # 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', # 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', # 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] # # ipsl_cm4 = ['ipsl_cm4'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ # 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', # 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', # 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] # # mpi_echam5 = ['mpi_echam5'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ # 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', # 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', # 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] # # miroc3_2_medres = ['miroc3_2_medres'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ # 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', # 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', # 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] # # cccma_cgmc3_1 = ['cccma_cgmc3'+str(scenario), 'http://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ # 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', # 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', # 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] # # miub_echo_g = ['miub_echo_g'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ # 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', # 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', # 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] # # mri_cgcm2_3_2a = ['mri_cgcm2_3_2a'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\ # 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly', # 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly', # 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly'] # if scenario == 'a1b': # data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mri_cgcm2_3_2a] # elif scenario == 'a2': # data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mpi_echam5,mri_cgcm2_3_2a] timestart = time.time() ################# #scenarios = ['a2', 'a1b'] scenario = 'a2' #for dataset in data: #file_loc = dir+'\\'+scenario+'\\'+dataset[0] file_loc = dir+'\\'+set if not os.path.exists(file_loc): os.mkdir(file_loc) os.chdir(file_loc) #print "The current scenario is: " +scenario + "dataset being worked on is: " + dataset[0] print "The current scenario is: " +scenario + "dataset being worked on is: " + set #The url of each dataset dataSet = vt_data[1] #dataSet = dataset[1] #The precipitation and temperatures of each dataset #Start at position(not index) 5 until the end of the #dictionary's key(which is a list) dataType=vt_data[5:] #dataType = dataset[5:] # daily or monthly for additional aggregation/formatting (not appended yet) timeStep = vt_data[2] length = timeStep #timeStep = dataset[2] #Start date timeBegin = vt_data[3] #timeBegin = dataset[3] #End date timeEnd = vt_data[4] #timeEnd = dataset[4] # data processing arguments gmlIDs=None verbose=True #coverage = 'false' check if on US border/ocean coverage = Region_return[2] delim='COMMA' stats='MEAN' # run the pyGDP start = time.time() outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats) end = time.time() print 'Start time is: ' + str(start) print 'End time is: ' + str(end) print 'Total time was: ' + str(end-start) print outputPath # copy the output and rename it # ind = 5 # for var in range(5, len(dataset)): # line = dataset[var].split('-') # dataset[ind] = line[4] # ind += 1 # # dataType = dataset[5:] #shutil.copy2(outputPath, region+'_'+dataset[0]+'.csv') shutil.copy2(outputPath, region+'_'+vt_data[0]+'.csv') # csvfile = os.getcwd()+region+'_'+dataset[0]+'.csv' # #Parse the csv file # index = 0 # # csvread = csv.reader(open(region+'_'+dataset[0] + '.csv', "rb")) # # csvwrite = csv.writer(open(dataType[0] + '.csv', "wb")) # # temp = csvread # var = temp.next() # var[0] = '#'+dataType[0] # #Gets gage ids # gage = temp.next() # # #Writes current variable to csv file # csvwrite.writerow(var) # #Writes all gage ids to csv file # csvwrite.writerow(gage) # # for variable in dataType: # # for row in csvread: # #if on last variable # if variable == dataType[len(dataType) - 1]: # csvwrite.writerow(row) # else: # if (row[0] in '#'+dataType[index+1]) or (row[0] in '# '+dataType[index+1]): # #Line 33 is used for titling the csv file the name of the variable (like tmin, ppt, or tmax) # var = '#'+dataType[index+1] # parsedFiles.append(os.getcwd()+'\\'+variable+'.csv') # csvwrite = csv.writer(open(dataType[index+1] + '.csv', "wb")) # row[1:] = "" # row[0] = var # csvwrite.writerow(row) # csvwrite.writerow(gage) # if len(dataType) == 2: # csvread.next() # else: # csvread.next() # csvwrite.writerow(csvread.next()) # csvwrite.writerow(csvread.next()) # break # else: # if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]: # csvwrite.writerow(row) # print "Finished parsing " + variable + ".csv" # parsedFiles.append(os.getcwd()+'\\'+variable+'.csv') # # use index to keep track of next variable # if (index + 1) < len(dataType): # index += 1 timeend = time.time() print 'Start time of pyGDP: ' + str(timestart) print 'End time of pyGDP: ' + str(timeend) print 'Total time of pyGDP: ' + str(timeend-timestart) os.chdir(dir)
# This dataset URI will be available here: http://cida.usgs.gov/thredds/ # At the time of creation, this dataset was internal USGS only. datasetURI = 'https://cida.usgs.gov/thredds/dodsC/ssebopeta/monthly' # Note that pyGDP offers convenience functions to determine what these options are. # These were derived from the thredds server housing the data. dataType = 'et' timeStart = '2000-01-01T00:00:00.000Z' timeEnd = '2010-01-01T00:00:00.000Z' outputFileName = 'ETAuSable2000-2010.csv' outputFile_handle = pyGDP.submitFeatureWeightedGridStatistics(shapefile, datasetURI, dataType, timeStart, timeEnd, attribute, value, verbose=True) os.rename(outputFile_handle, outputFileName) dates = [] vals = [] datesT, valsT = numpy.loadtxt( outputFileName, unpack=True, skiprows=3, delimiter=',', converters={0: mdates.strpdate2num("%Y-%m-%dT%H:%M:%SZ")}) for v in valsT:
import pyGDP import matplotlib.dates as mdates import numpy as np pyGDP = pyGDP.pyGDPwebProcessing() """ This example shows how easy it is to make a call, if all inputs are known before hand. """ shapefile = 'sample:simplified_HUC8s' user_attribute = 'REGION' user_value = 'Great Lakes Region' dataSet = 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml' dataType = 'Prcp' timeBegin = '1970-01-23T00:00:00.000Z' timeEnd = '1979-01-23T00:00:00.000Z' print 'Processing request.' outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value) jd,precip =np.loadtxt(outputPath,unpack=True,skiprows=3,delimiter=',', converters={0: mdates.strpdate2num('%Y-%m-%dT%H:%M:%SZ')}) print precip[0:100]
import numpy as np pyGDP = pyGDP.pyGDPwebProcessing() """ This example shows how easy it is to make a call, if all inputs are known before hand. """ shapefile = 'sample:simplified_HUC8s' user_attribute = 'REGION' user_value = 'Great Lakes Region' dataSet = 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml' dataType = 'Prcp' timeBegin = '1970-01-23T00:00:00.000Z' timeEnd = '1979-01-23T00:00:00.000Z' print 'Processing request.' outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value) jd, precip = np.loadtxt( outputPath, unpack=True, skiprows=3, delimiter=',', converters={0: mdates.strpdate2num('%Y-%m-%dT%H:%M:%SZ')}) print precip[0:100]
# as an input instead of value. michGMLID = pyGDP.getGMLIDs(shapefile, usr_attribute, 'Michigan') gmlIDs = michGMLID # We get the dataset URI that we are interested in by searching for prism: dataSetURIs = pyGDP.getDataSetURI(anyText='prism') pp = pprint.PrettyPrinter(indent=5, width=60) pp.pprint(dataSetURIs) # Set our datasetURI dataSetURI = 'dods://cida.usgs.gov/thredds/dodsC/prism' # Get the available data types associated with the dataset datatypes = pyGDP.getDataType(dataSetURI) for dt in datatypes: print(dt) # Set the dataType. Note that leaving dataType out below will select all. dataType = 'ppt' # Get available time range on the dataset timeRange = pyGDP.getTimeRange(dataSetURI, dataType) for t in timeRange: print(t) # Instead of submitting in a value, we submit a list of gmlIDs associated # with either a small portion of that value, or multiple values. value = None path = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSetURI, dataType, timeRange[0], timeRange[0], usr_attribute, value, gmlIDs, verbose=True) print(path)
#We set our value to 5 usr_value = 5 # our shapefile = 'upload:OKCNTYD', usr_attribute = 'OBJECTID', and usr_value = 13 # We get the dataset URI that we are interested in dataSetURIs = pyGDP.getDataSetURI(anyText='prism') pp = pprint.PrettyPrinter(indent=5,width=60) pp.pprint(dataSetURIs) print # Set our datasetURI dataSetURI = 'dods://cida.usgs.gov/thredds/dodsC/prism' # Get the available data types associated with the dataset dataType = 'ppt' # Get available time range on the dataset timeRange = pyGDP.getTimeRange(dataSetURI, dataType) for t in timeRange: print t timeBegin = '1900-01-01T00:00:00.000Z' timeEnd = '1901-01-01T00:00:00.000Z' print textFile = pyGDP.submitFeatureWeightedGridStatistics(OKshapefile, dataSetURI, dataType, timeBegin, timeEnd,usr_attribute, usr_value,verbose=True) jd,precip=np.loadtxt(textFile,unpack=True,skiprows=3,delimiter=',', converters={0: mdates.strpdate2num('%Y-%m-%dT%H:%M:%SZ')}) print 'Some data:' print precip[0:100]
def main_func(): def Region_lookup(region): region_properties = [] if region == 'nhru': region_properties.append(371) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d35da1e4b0ca184833940c') region_properties.append('false') if region == 'R01': region_properties.append(2462) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5244735ae4b05b217bada04e') region_properties.append('false') if region == 'R02': region_properties.append(4827) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52696784e4b0584cbe9168ee') region_properties.append('false') if region == 'R03': region_properties.append(9899) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5283bd23e4b047efbbb57922') region_properties.append('false') if region == 'R04': region_properties.append(5936) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5284ff57e4b063f258e61b9d') region_properties.append('false') if region == 'R05': region_properties.append(7182) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/528516bbe4b063f258e62161') region_properties.append('false') if region == 'R06': region_properties.append(2303) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d75296e4b055e0afd5be2c') region_properties.append('true') if region == 'R07': region_properties.append(8205) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52851cd5e4b063f258e643dd') region_properties.append('true') if region == 'R08': region_properties.append(4449) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52854695e4b063f258e6513c') region_properties.append('true') if region == 'R10L': region_properties.append(8603) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/520031dee4b0ad2d97189db2') region_properties.append('true') elif region =='R10U': region_properties.append(10299) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5216849ee4b0b45d6ba61e2e') region_properties.append('false') elif region == 'R11': region_properties.append(7373) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d1f9ebe4b08b18a62d586b') region_properties.append('true') elif region == 'R12': region_properties.append(7815) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5201328ae4b009d47a4c247a') region_properties.append('false') elif region == 'R13': region_properties.append(1958) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d752b9e4b055e0afd5be36') region_properties.append('false') elif region == 'R14': region_properties.append(3879) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52029c68e4b0e21cafa4b40c') region_properties.append('false') elif region == 'R15': region_properties.append(3441) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5285389ae4b063f258e64863') region_properties.append('false') elif region == 'R16': region_properties.append(2664) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52853f97e4b063f258e64875') region_properties.append('false') return region_properties global parsedFiles, csvfile, vistrails_data_set, nhru, length regions = ['R06', 'R10L', 'R11', 'R12', 'R13', 'R14'] region = 'R06' parsedFiles = [] csvfile = '' vistrails_data_set = [] Region_return=Region_lookup(region) hrus = Region_return[0] nhru = hrus ScienceBase_URL= Region_return[1] pyGDP.WFS_URL = ScienceBase_URL # call web processing module pyGDP = pyGDP.pyGDPwebProcessing() # change working directory so the GDP output will be written there # Datasets and their properties # run 1, 2 , and 3 #********************************** # run 1 only #scenario = 'a2' # other scenarios are 'a1b' and 'b1' scenarios = ['a2','a1b'] # not running b1 or a1fi timestart = '1960-01-15T00:00:00.000Z' timeend = '2010-12-15T00:00:00.000Z' # timestart = '2010-12-15T00:00:00.000Z' # timeend='2099-12-15T00:00:00.000Z' #********************************** # datasets greyed out - we are not initially using. ##ccsm = ['ccsm'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ ## 'ccsm-'+scenarios[0]+'-pr-NAm-grid', ## 'ccsm-'+scenarios[0]+'-tmax-NAm-grid', ## 'ccsm-'+scenarios[0]+'-tmin-NAm-grid'] shapefiles = pyGDP.getShapefiles() for shp in shapefiles: print shp # feature loaded from sciencebase #should shapefile be sb:SP_hru instead? shapefile = 'sb:'+region+'_hru' user_attribute = 'hru_id_loc' user_value = None os.chdir('C:\\Users\\reimandy\\workspace\\userpackages\\WaterBalanceModel\\Step1_CLIMATE_DATA\\'+region+'\\HAYHOE') dir = os.getcwd() for scenario in scenarios: cgcm3_t47 = ['cgcm3_t47'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ 'cgcm3_t47-'+str(scenario)+'-pr-NAm-grid', 'cgcm3_t47-'+str(scenario)+'-tmax-NAm-grid', 'cgcm3_t47-'+str(scenario)+'-tmin-NAm-grid'] cgcm3_t63 = ['cgcm3_t63'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ 'cgcm3_t63-'+str(scenario)+'-pr-NAm-grid', 'cgcm3_t63-'+str(scenario)+'-tmax-NAm-grid', 'cgcm3_t63-'+str(scenario)+'-tmin-NAm-grid'] cnrm = ['cnrm'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ 'cnrm-'+str(scenario)+'-pr-NAm-grid', 'cnrm-'+str(scenario)+'-tmax-NAm-grid', 'cnrm-'+str(scenario)+'-tmin-NAm-grid'] ##csiro = ['csiro'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ ## 'csiro-'+scenarios[0]+'-pr-NAm-grid', ## 'csiro-'+scenarios[0]+'-tmax-NAm-grid', ## 'csiro-'+scenarios[0]+'-tmin-NAm-grid'] ##echam5 = ['echam5'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ ## 'echam5-'+scenarios[0]+'-pr-NAm-grid', ## 'echam5-'+scenarios[0]+'-tmax-NAm-grid', ## 'echam5-'+scenarios[0]+'-tmin-NAm-grid'] echo = ['echo'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ 'echo-'+str(scenario)+'-pr-NAm-grid', 'echo-'+str(scenario)+'-tmax-NAm-grid', 'echo-'+str(scenario)+'-tmin-NAm-grid'] gfdl_2_1 = ['gfdl_2-1'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ 'gfdl_2-1-'+str(scenario)+'-pr-NAm-grid', 'gfdl_2-1-'+str(scenario)+'-tmax-NAm-grid', 'gfdl_2-1-'+str(scenario)+'-tmin-NAm-grid'] ##giss_aom = ['giss_aom'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ ## 'giss_aom-'+scenarios[0]+'-pr-NAm-grid', ## 'giss_aom-'+scenarios[0]+'-tmax-NAm-grid', ## 'giss_aom-'+scenarios[0]+'-tmin-NAm-grid'] ##hadcm3 = ['hadcm3'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ ## 'hadcm3-'+scenarios[0]+'-pr-NAm-grid', ## 'hadcm3-'+scenarios[0]+'-tmax-NAm-grid', ## 'hadcm3-'+scenarios[0]+'-tmin-NAm-grid'] ##miroc_hi = ['miroc_hi'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ ## 'miroc_hi-'+scenarios[0]+'-pr-NAm-grid', ## 'miroc_hi-'+scenarios[0]+'-tmax-NAm-grid', ## 'miroc_hi-'+scenarios[0]+'-tmin-NAm-grid'] miroc_med = ['miroc_med'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ 'miroc_med-'+str(scenario)+'-pr-NAm-grid', 'miroc_med-'+str(scenario)+'-tmax-NAm-grid', 'miroc_med-'+str(scenario)+'-tmin-NAm-grid'] mri_cgcm2 = ['mri_cgcm2'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ 'mri_cgcm2-'+str(scenario)+'-pr-NAm-grid', 'mri_cgcm2-'+str(scenario)+'-tmax-NAm-grid', 'mri_cgcm2-'+str(scenario)+'-tmin-NAm-grid'] ##pcm = ['pcm'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\ ## 'pcm-'+scenarios[0]+'-pr-NAm-grid', ## 'pcm-'+scenarios[0]+'-tmax-NAm-grid', ## 'pcm-'+scenarios[0]+'-tmin-NAm-grid'] # get list of shapefiles uploaded to the GDP if scenario == 'a1b': #data=[cgcm3_t47,cgcm3_t63,cnrm,echam5,echo,giss_aom,hadcm3,miroc_hi,pcm] data=[cgcm3_t47,cgcm3_t63,cnrm,echo] elif scenario == 'a2': #data=[ccsm,cgcm3_t47,cgcm3_t63,cnrm,csiro,echam5,echo,gfdl_2_1,hadcm3,miroc_med,mri_cgcm2,pcm] data = [cgcm3_t47,cgcm3_t63,cnrm,echo,gfdl_2_1,miroc_med,mri_cgcm2] for dataset in data: file_loc = dir+'\\'+scenario+'\\'+dataset[0] if not os.path.exists(file_loc): os.mkdir(file_loc) os.chdir(file_loc) print "The current dataset being worked on is: " + dataset[0] #The url of each dataset dataSet = dataset[1] #The precipitation and temperatures of each dataset #Start at position(not index) 5 until the end of the #dictionary's key(which is a list) dataType = dataset[5:] # http://cida.usgs.gov/thredds/dodsC/dcp/conus or Daily for additional aggregation/formatting (not appended yet) timeStep = dataset[2] #Start date timeBegin = dataset[3] #End date timeEnd = dataset[4] # data processing arguments gmlIDs=None verbose=True #coverage = 'false' check if on US border/ocean coverage = Region_return[2] delim='COMMA' stats='MEAN' # run the pyGDP start = time.time() outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats) end = time.time() print 'Start time is: ' + str(start) print 'End time is: ' + str(end) print 'Total time was: ' + str(end-start) print outputPath # copy the output and rename it shutil.copy2(outputPath, region+'_'+dataset[0]+'.csv') ind = 5 for var in range(5, len(dataset)): line = dataset[var].split('-') dataset[ind] = line[2] ind += 1 dataType = dataset[5:] #Parse the csv file index = 0 #parsedFiles = [] csvread = csv.reader(open(region+'_'+dataset[0] + '.csv', "rb")) csvwrite = csv.writer(open(dataType[0] + '.csv', "wb")) #parsedFiles.append(csvwrite) temp = csvread var = temp.next() var[0] = '#'+dataType[0] #Gets gage ids gage = temp.next() #Writes current variable to csv file csvwrite.writerow(var) #Writes all gage ids to csv file csvwrite.writerow(gage) for variable in dataType: for row in csvread: #if on last variable if variable == dataType[len(dataType) - 1]: csvwrite.writerow(row) else: if (row[0] in '#'+dataType[index+1]) or (row[0] in '# '+dataType[index+1]): #Line 33 is used for titling the csv file the name of the variable (like tmin, ppt, or tmax) var = '#'+dataType[index+1] csvwrite = csv.writer(open(dataType[index+1] + '.csv', "wb")) #parsedFiles.append(csvwrite) row[1:] = "" row[0] = var csvwrite.writerow(row) csvwrite.writerow(gage) if len(dataType) == 2: csvread.next() else: temp = csvread csvwrite.writerow(csvread.next()) csvwrite.writerow(temp.next()) break else: if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]: csvwrite.writerow(row) print "Finished parsing " + variable + ".csv" # use index to keep track of next variable if (index + 1) < len(dataType): index += 1 #pyGDP.setCSV_Parsed_Files(parsedFiles) os.chdir(dir)
def main_func(curdir, data_set, region): def Region_lookup(region): region_properties = [] if region == "nhru": region_properties.append(371) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/51d35da1e4b0ca184833940c") region_properties.append("false") elif region == "R01": region_properties.append(2462) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5244735ae4b05b217bada04e") region_properties.append("false") elif region == "R02": region_properties.append(4827) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52696784e4b0584cbe9168ee") region_properties.append("false") elif region == "R03": region_properties.append(9899) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5283bd23e4b047efbbb57922") region_properties.append("false") elif region == "R04": region_properties.append(5936) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5284ff57e4b063f258e61b9d") region_properties.append("false") elif region == "R05": region_properties.append(7182) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/528516bbe4b063f258e62161") region_properties.append("false") elif region == "R06": region_properties.append(2303) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/51d75296e4b055e0afd5be2c") region_properties.append("true") elif region == "R07": region_properties.append(8205) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52851cd5e4b063f258e643dd") region_properties.append("true") elif region == "R08": region_properties.append(4449) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52854695e4b063f258e6513c") region_properties.append("true") elif region == "R10L": region_properties.append(8603) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/520031dee4b0ad2d97189db2") region_properties.append("true") elif region == "R10U": region_properties.append(10299) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5216849ee4b0b45d6ba61e2e") region_properties.append("false") elif region == "R11": region_properties.append(7373) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/51d1f9ebe4b08b18a62d586b") region_properties.append("true") elif region == "R12": region_properties.append(7815) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5201328ae4b009d47a4c247a") region_properties.append("false") elif region == "R13": region_properties.append(1958) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/51d752b9e4b055e0afd5be36") region_properties.append("false") elif region == "R14": region_properties.append(3879) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52029c68e4b0e21cafa4b40c") region_properties.append("false") elif region == "R15": region_properties.append(3441) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5285389ae4b063f258e64863") region_properties.append("false") elif region == "R16": region_properties.append(2664) region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52853f97e4b063f258e64875") region_properties.append("false") return region_properties def list_define(data_set): if data_set == "PRISM": return [ "PRISM", "http://cida.usgs.gov/thredds/dodsC/prism", "Monthly", "2010-01-01T00:00:00.000Z", "2012-12-31T00:00:00.000Z", "ppt", "tmx", "tmn", ] elif data_set == "MPI_ECHAM5": return [ "MPI ECHAM5", "http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/EH5/merged/Monthly/RegCM3_Monthly_merged_EH5.ncml", "Monthly", "1968-01-01T00:00:00.000Z", "2099-12-31T00:00:00.000Z", "RT", "TA", ] elif data_set == "GENMON": return [ "GENMON", "http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/GENMOM/merged/Monthly/RegCM3_Monthly_merged_GENMOM.ncml", "Monthly", "1980-01-01T00:00:00.000Z", "2089-12-31T00:00:00.000Z", "RT", "TA", ] elif data_set == "GFDL_CM_2_0": return [ "GFDL CM 2.0", "http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/GFDL/merged/Monthly/RegCM3_Monthly_merged_GFDL.ncml", "Monthly", "1970-01-01T00:00:00.000Z", "2069-12-31T00:00:00.000Z", "RT", "TA", ] elif data_set == "NOAA_NCEP": return [ "NOAA NCEP", "http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/NCEP/merged/Monthly/RegCM3_Monthly_merged_NCEP.ncml", "Monthly", "1982-01-01T00:00:00.000Z", "2007-12-31T00:00:00.000Z", "RT", "TA", ] elif data_set == "GSD": return [ "Gridded Observed Data(1949-2010)", "http://cida.usgs.gov/thredds/dodsC/new_gmo", "Daily", "1949-01-01T00:00:00.000Z", "2010-12-31T00:00:00.000Z", "pr", "tas", ] elif data_set == "DAYMET": return [ "DAYMET", "dods://cida-eros-mows1.er.usgs.gov:8080/thredds/dodsC/daymet", "Daily", "2010-01-01T00:00:00.000Z", "2012-01-01T00:00:00.000Z", "prcp", "tmax", "tmin", ] elif data_set == "Idaho": return [ "Idaho", "http://cida.usgs.gov/thredds/dodsC/UofIMETDATA", "Daily", "1979-01-01T00:00:00.000Z", "2013-01-01T00:00:00.000Z", "precipitation_amount", "min_air_temperature", "max_air_temperature", ] global csvfile, vt_dataset, nhru, length, vt_datatype, url import pyGDP Region_return = Region_lookup(region) hrus = Region_return[0] nhru = hrus ScienceBase_URL = Region_return[1] # NHDplus region # link to region 6 hrus on ScienceBase pyGDP.WFS_URL = ScienceBase_URL url = pyGDP.WFS_URL pyGDP = pyGDP.pyGDPwebProcessing() # change working directory so the GDP output will be written there # Datasets and their properties # ********************************** # Gridded_Observed_Data_1950_1999 = ['Gridded Observed Data(1950-1999)', 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml', 'Daily', '1950-01-01T00:00:00.000Z', '1999-12-31T00:00:00.000Z', 'Prcp', 'Tavg'] # data = [PRISM]#,MPI_ECHAM5,GENMON,GFDL_CM_2_0,NOAA_NCEP,GSD,DAYMET] # prism starts at 1895 # get list of shapefiles uploaded to the GDP shapefiles = pyGDP.getShapefiles() for shp in shapefiles: print shp # feature loaded from sciencebase # shapefile = 'sb:'+region+'_hru' shapefile = "sb:nhru" user_attribute = "hru_id_loc" user_value = None # os.chdir('C:\\Users\\reimandy\\workspace\\userpackages\\WaterBalanceModel\\Step1_CLIMATE_DATA\\'+region+'\\SBDDS') os.chdir(curdir) dir = os.getcwd() vt_data = list_define(data_set) vt_datatype = vt_data[5:] timestart = time.time() file_loc = curdir + "\\" + data_set if not os.path.exists(file_loc): os.mkdir(file_loc) os.chdir(file_loc) print "The current dataset being worked on is: " + data_set # The url of each dataset dataSet = vt_data[1] # The precipitation and temperatures of each dataset # Start at position(not index) 5 until the end of the # dictionary's key(which is a list) dataType = vt_data[5:] # daily or monthly for additional aggregation/formatting (not appended yet) timeStep = vt_data[2] # Length is for connecting to vistrails length = timeStep # Start date timeBegin = vt_data[3] # End date timeEnd = vt_data[4] # data processing arguments gmlIDs = None verbose = True # coverage = 'false' check if on US border/ocean coverage = Region_return[2] delim = "COMMA" stats = "MEAN" # run the pyGDP start = time.time() outputPath = pyGDP.submitFeatureWeightedGridStatistics( shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats, ) end = time.time() print "Start time is: " + str(start) print "End time is: " + str(end) print "Total time was: " + str(end - start) print outputPath # copy the output and rename it shutil.copy2(outputPath, region + "_" + vt_data[0] + ".csv") # #Parse the csv file # index = 0 # # csvread = csv.reader(open(region+'_'+dataset[0] + '.csv', "rb")) # csvwrite = csv.writer(open(dataType[0] + '.csv', "wb")) # # index = 0 # # temp = csvread # var = temp.next() # #Gets gage ids # gage = temp.next() # # #Writes current variable to csv file # csvwrite.writerow(var) # #Writes all gage ids to csv file # csvwrite.writerow(gage) # # for variable in dataType: # # for row in csvread: # #if on last variable # if variable == dataType[len(dataType) - 1]: # csvwrite.writerow(row) # else: # if (row[0] in '#'+dataType[index+1]) or (row[0] in '# '+dataType[index+1]): # #Line 33 is used for titling the csv file the name of the variable (like tmin, ppt, or tmax) # var = '#'+dataType[index+1] # csvwrite = csv.writer(open(dataType[index+1] + '.csv', "wb")) # row[1:] = "" # row[0] = var # csvwrite.writerow(row) # csvwrite.writerow(gage) # # if len(dataType) == 2: # csvread.next() # else: # csvread.next() # csvwrite.writerow(csvread.next()) # csvwrite.writerow(csvread.next()) # break # else: # if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]: # csvwrite.writerow(row) # print "Finished parsing " + variable + ".csv" # parsedFiles.append(os.getcwd()+'\\'+variable+'.csv') # # use index to keep track of next variable # if (index + 1) < len(dataType): # index += 1 timeend = time.time() print "Start time of pyGDP: " + str(timestart) print "End time of pyGDP: " + str(timeend) print "Total time of pyGDP: " + str(timeend - timestart) os.chdir(dir)
def main_func(region): #NHDplus region region = 'R06' #region = pyGDP.getRegion() # change working directory so the GDP output will be written there # Datasets and their properties #********************************** Gridded_Observed_Data_1950_1999 = ['Gridded Observed Data(1950-1999)', 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml', 'Daily', '1950-01-01T00:00:00.000Z', '1999-12-31T00:00:00.000Z', 'Prcp', 'Tavg'] Gridded_Observed_Data_1949_2010 = ['Gridded Observed Data(1949-2010)', 'http://cida.usgs.gov/thredds/dodsC/new_gmo', 'Daily', '1950-01-01T00:00:00.000Z', '1950-02-01T00:00:00.000Z', 'pr', 'tas'] PRISM = ['PRISM', 'http://cida.usgs.gov/thredds/dodsC/prism', 'Monthly', '1895-01-01T00:00:00.000Z', '2012-12-31T00:00:00.000Z', 'ppt', 'tmx', 'tmn'] DAYMET = ['DAYMET', 'dods://cida-eros-mows1.er.usgs.gov:8080/thredds/dodsC/daymet', 'Daily', '1980-01-01T00:00:00.000Z', '2012-01-01T00:00:00.000Z', 'prcp', 'tmax', 'tmin'] MPI_ECHAM5 = ['MPI ECHAM5', 'http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/EH5/merged/Monthly/RegCM3_Monthly_merged_EH5.ncml', 'Monthly', '1968-01-01T00:00:00.000Z', '2099-12-31T00:00:00.000Z', 'RT', 'TA'] GENMON = ['GENMON', 'http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/GENMOM/merged/Monthly/RegCM3_Monthly_merged_GENMOM.ncml', 'Monthly', '1980-01-01T00:00:00.000Z', '2089-12-31T00:00:00.000Z', 'RT', 'TA'] GFDL_CM_2_0 = ['GFDL CM 2.0', 'http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/GFDL/merged/Monthly/RegCM3_Monthly_merged_GFDL.ncml', 'Monthly', '1980-01-01T00:00:00.000Z', '2069-12-31T00:00:00.000Z', 'RT', 'TA'] NOAA_NCEP = ['NOAA NCEP', 'http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/NCEP/merged/Monthly/RegCM3_Monthly_merged_NCEP.ncml','Monthly', '1982-01-01T00:00:00.000Z', '2007-12-31T00:00:00.000Z','RT','TA'] #List dataset names data = [Gridded_Observed_Data_1950_1999, Gridded_Observed_Data_1949_2010,PRISM,DAYMET,MPI_ECHAM5,GENMON,GFDL_CM_2_0,NOAA_NCEP] #data=[Gridded_Observed_Data_1950_1999] # link to region 6 hrus on ScienceBase # put a function here to retrieve sciencebase url and number of hrus for specific region pyGDP.WFS_URL = 'https://www.sciencebase.gov/catalogMaps/mapping/ows/51b0d374e4b030b519830d73' #Get pyGDP.WFS_URL from vistrials #pyGDP.WFS_URL = pyGDP.get_pyGDP_WFS_URL() # region 6 = 2303 hrus # for more info read: # science base access: https://my.usgs.gov/confluence/display/GeoDataPortal/pyGDP+FAQ # call web processing module # will automatically use shapefiles from GDP as default unless # you define web service pyGDP = pyGDP.pyGDPwebProcessing() # get list of shapefiles uploaded to the GDP shapefiles = pyGDP.getShapefiles() for shp in shapefiles: print shp # feature loaded from sciencebase # automate region number shapefile = 'sb:R06a_hru' # shapefile/feature attribute for which you are summarizing info user_attribute = 'hru_id_loc' # create list of hru_ids that can be used for the uservalue # need to automate number of hurs in shapefile, maybe include from scienecbase # url retrieval above lin 25-29 # single feature id test case #user_value = '10' # to summarize for all ids user_value = None for dataset in data: print "The current dataset being worked on is: " + dataset[0] #The url of each dataset dataSet = dataset[1] #The precipitation and temperatures of each dataset #Start at position(not index) 5 until the end of the #dictionary's key(which is a list) dataType=dataset[5:] # daily or monthly for additional aggregation/formatting (not appended yet) timeStep = dataset[2] #Start date timeBegin = dataset[3] #End date timeEnd = dataset[4] # data processing arguments gmlIDs=None verbose=True #coverage = 'false' check if on US border/ocean coverage='true' delim='COMMA' stats='MEAN' # run the pyGDP outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats) print outputPath #pyGDP.setCSV_file(outputPath) # copy the output and rename it shutil.copy2(outputPath, region+'_'+dataset[0]+'.csv') csvfile = os.getcwd()+region+'_'+dataset[0]+'.csv' #Parse the csv file index = 0 #parsedFiles = [] csvread = csv.reader(open(outputPath + '.csv', "rb")) csvwrite = csv.writer(open(dataType[0] + '.csv', "wb")) temp = csvread var = temp.next() #Gets gage ids gage = temp.next() #Writes current variable to csv file csvwrite.writerow(var) #Writes all gage ids to csv file csvwrite.writerow(gage) for variable in dataType: for row in csvread: #if on last variable if variable == dataType[len(dataType) - 1]: csvwrite.writerow(row) else: if (row[0] in '#'+dataType[index+1]) or (row[0] in '# '+dataType[index+1]): #Line 33 is used for titling the csv file the name of the variable (like tmin, ppt, or tmax) var = '#'+dataType[index+1] parsedFiles.append(os.getcwd()+'\\'+variable+'.csv') csvwrite = csv.writer(open(dataType[index+1] + '.csv', "wb")) row[1:] = "" row[0] = var csvwrite.writerow(row) csvwrite.writerow(gage) if len(dataType) == 2: csvread.next() else: temp = csvread csvwrite.writerow(csvread.next()) csvwrite.writerow(temp.next()) break else: if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]: csvwrite.writerow(row) print "Finished parsing " + variable + ".csv" parsedFiles.append(os.getcwd()+'\\'+variable+'.csv') # use index to keep track of next variable if (index + 1) < len(dataType): index += 1
# 'http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/ncml/days_prcp_abv_cmip5_hist_der.ncml', # 'http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/ncml/cooling_degree_day_cmip5_hist_der.ncml'] for dataURI in dataURIs: remote_dataURI=dataURI.replace('http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/','http://cida-eros-thredds3.er.usgs.gov:8080/thredds/dodsC/cmip5_bcca/derivatives/') print(remote_dataURI) dataTypes = pyGDP.getDataType(remote_dataURI) timeRange = pyGDP.getTimeRange(remote_dataURI, dataTypes[0]) if len(dataTypes)==1: dataTypes=dataTypes[0] for shapefile in shapefiles.keys(): outputfilename=shapefile.replace('derivative:','')+'_'+dataURI.replace('http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/ncml/','') outputfilename=outputfilename.replace('http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/averages_rcp/ncml/','') outputfilename=outputfilename.replace('http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/averages_hist/ncml/','') outputfilename=outputfilename.replace('http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/averages_gmo/ncml/','') outputfilename=outputfilename.replace('http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/cmip5_obs_der/','') if not os.path.isfile(outputfilename): open(outputfilename, 'a').close() print shapefile print dataURI print dataTypes print timeRange print shapefiles[shapefile] outFile=pyGDP.submitFeatureWeightedGridStatistics(geoType=shapefile, dataSetURI=dataURI, varID=dataTypes, startTime=timeRange[0], endTime=timeRange[1], attribute=shapefiles[shapefile], coverage=False, outputfname=outputfilename)
value='Au Sable' ### Land Use Land Cover # This dataset URI will be available here: http://cida.usgs.gov/thredds/ # At the time of creation, this dataset was internal USGS only. datasetURI='https://cida.usgs.gov/thredds/dodsC/ssebopeta/monthly' # Note that pyGDP offers convenience functions to determine what these options are. # These were derived from the thredds server housing the data. dataType = 'et' timeStart = '2000-01-01T00:00:00.000Z' timeEnd = '2010-01-01T00:00:00.000Z' outputFileName = 'ETAuSable2000-2010.csv' outputFile_handle = pyGDP.submitFeatureWeightedGridStatistics(shapefile, datasetURI, dataType, timeStart, timeEnd, attribute, value,verbose=True) os.rename(outputFile_handle,outputFileName) dates = [] vals = [] datesT,valsT = numpy.loadtxt(outputFileName,unpack=True,skiprows=3,delimiter=',', converters={0: mdates.strpdate2num("%Y-%m-%dT%H:%M:%SZ")}) for v in valsT: vals.append(v) for d in datesT: dates.append(d) fig = pylab.figure(figsize=(12,6),facecolor='w') fig.suptitle('SSEBop Actual Evapotranspiration, Au Sable Watershed',fontsize=26)
# Set our datasetURI dataSetURI = 'dods://cida.usgs.gov/thredds/dodsC/prism' # Get the available data types associated with the dataset dataType = 'ppt' # Get available time range on the dataset timeRange = pyGDP.getTimeRange(dataSetURI, dataType) for t in timeRange: print t timeBegin = '1900-01-01T00:00:00.000Z' timeEnd = '1901-01-01T00:00:00.000Z' print textFile = pyGDP.submitFeatureWeightedGridStatistics(OKshapefile, dataSetURI, dataType, timeBegin, timeEnd, usr_attribute, usr_value, verbose=True) jd, precip = np.loadtxt( textFile, unpack=True, skiprows=3, delimiter=',', converters={0: mdates.strpdate2num('%Y-%m-%dT%H:%M:%SZ')}) print 'Some data:' print precip[0:100]
outputpath_earlys = list() outputpath_lates = list() for datatype in datatypes: if 'rcp85' in datatype: #if 'tasmax' in datatype and 'rcp85' in datatype: print(datatype) outName = 'Loca_early_Future_' + datatype + '.csv' if os.path.isfile(outName): outputpath_earlys.append(outputPath) else: outputPath = pyGDP.submitFeatureWeightedGridStatistics( shapefile, dataSetURI, datatype, '2006-01-01T00:00:00Z', '2009-12-31T00:00:00Z', polyAttribute, polyValue, outputfname=outName) print(outputPath) outputpath_earlys.append(outputPath) outName = 'Loca_late_Feature_' + datatype + '.csv' if os.path.isfile(outName): outputpath_lates.append(outputPath) else: outputPath = pyGDP.submitFeatureWeightedGridStatistics( shapefile, dataSetURI, datatype, '2096-01-01T00:00:00Z',
# In[9]: # Set the dataType. Note that leaving dataType out below will select all. # Get available time range on the dataset dataType='prcp' timeRange = pyGDP.getTimeRange(dataSetURI, dataType) for t in timeRange: print t # In[14]: # Submit job to pyGDP # result is area-weighted mean in CSV for given datatype path = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSetURI, dataType, timeRange[0], timeRange[1], usr_attribute, usr_values, gmlIDs=None, verbose=True) print path import shutil shutil.copy2(path, names+'_'+dataType) # In[20]: datatype="tmax" path = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSetURI, dataType, timeRange[0], timeRange[1], usr_attribute, usr_values, gmlIDs=None, verbose=True) print path import shutil shutil.copy2(path, names+'_'+dataType) # In[17]:
# Grab the file and get its attributes: shapefile = 'sample:CSC_Boundaries' attributes = pyGDP.getAttributes(shapefile) for attr in attributes: print attr # Grab the values from 'area_name' and 'sample:CSC_Boundaries' usr_attribute = 'area_name' values = pyGDP.getValues(shapefile, usr_attribute) for v in values: print v usr_value = 'Southwest' dataSetURI = 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml' dataTypes = pyGDP.getDataType(dataSetURI) for d in dataTypes: print d dataType = 'Prcp' # Get available time range on the dataset timeRange = pyGDP.getTimeRange(dataSetURI, dataType) for t in timeRange: print t timeBegin = '1960-01-01T00:00:00.000Z' timeEnd = '1960-01-21T00:00:00.000Z' outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSetURI, dataType, timeBegin, timeEnd, usr_attribute, usr_value, gmlIDs=None, verbose=True)
print # Grab the values from 'OBJECTID' and 'upload:OKCNTYD' usr_attribute = 'STATE' values = pyGDP.getValues(shapefile, usr_attribute) for v in values: print v print # our shapefile = 'upload:OKCNTYD', usr_attribute = 'OBJECTID', and usr_value = 13 # We get the dataset URI that we are interested in dataSetURIs = pyGDP.getDataSetURI() for d in dataSetURIs: print d # Set our datasetURI dataSetURI = 'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/gmo/GMO_w_meta.ncml' # Get the available data types associated with the dataset dataType = 'Prcp' # Get available time range on the dataset timeRange = pyGDP.getTimeRange(dataSetURI, dataType) for t in timeRange: print t """ Instead of submitting in a value, we submit a list of gmlIDs associated with either a small portion of that value, or multiple values. """ values = ['Wisconsin', 'Michigan', 'Minnesota'] path = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSetURI, dataType, timeRange[0], timeRange[0], usr_attribute, values) print path
fileList = [] for yr in years: datasetURI = '%s%s%s' % (baseURI, yr, '.nc') timeRange = pyGDP.getTimeRange(datasetURI, varID) timeStart = timeRange[0] timeEnd = timeRange[1] print('begining process request for %s on year %s' % (varID, yr)) outputFile_handle = pyGDP.submitFeatureWeightedGridStatistics( userPoly, datasetURI, varID, timeStart, timeEnd) fileList.append(outputFile_handle) fig = pylab.figure(figsize=(12, 6), facecolor='w') ax = fig.gca() ax.set_ylabel('Air Temperature (C)') ax.plot_date(dates, vals, 'b-') ax.xaxis.set_major_locator(mdates.YearLocator(5, month=1, day=1)) ax.xaxis.set_major_formatter(mdates.DateFormatter(' %Y'))
def main_func(region, currdir, timestart, timeend):#, scenarios): def Region_lookup(region): region_properties = [] if region == 'nhru': region_properties.append(371) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d35da1e4b0ca184833940c') region_properties.append('false') elif region == 'R01': region_properties.append(2462) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5244735ae4b05b217bada04e') region_properties.append('false') elif region == 'R02': region_properties.append(4827) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52696784e4b0584cbe9168ee') region_properties.append('false') elif region == 'R03': region_properties.append(9899) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5283bd23e4b047efbbb57922') region_properties.append('false') elif region == 'R04': region_properties.append(5936) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5284ff57e4b063f258e61b9d') region_properties.append('false') elif region == 'R05': region_properties.append(7182) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/528516bbe4b063f258e62161') region_properties.append('false') elif region == 'R06': region_properties.append(2303) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d75296e4b055e0afd5be2c') region_properties.append('true') elif region == 'R07': region_properties.append(8205) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52851cd5e4b063f258e643dd') region_properties.append('true') elif region == 'R08': region_properties.append(4449) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52854695e4b063f258e6513c') region_properties.append('true') elif region == 'R10L': region_properties.append(8603) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/520031dee4b0ad2d97189db2') region_properties.append('true') elif region =='R10U': region_properties.append(10299) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5216849ee4b0b45d6ba61e2e') region_properties.append('false') elif region == 'R11': region_properties.append(7373) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d1f9ebe4b08b18a62d586b') region_properties.append('true') elif region == 'R12': region_properties.append(7815) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5201328ae4b009d47a4c247a') region_properties.append('false') elif region == 'R13': region_properties.append(1958) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d752b9e4b055e0afd5be36') region_properties.append('false') elif region == 'R14': region_properties.append(3879) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52029c68e4b0e21cafa4b40c') region_properties.append('false') elif region == 'R15': region_properties.append(3441) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5285389ae4b063f258e64863') region_properties.append('false') elif region == 'R16': region_properties.append(2664) region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52853f97e4b063f258e64875') region_properties.append('false') return region_properties global parsedFiles, csvfile, vistrails_data_set, nhru, url import pyGDP #region = 'R13' parsedFiles = [] vistrails_data_set = [] Region_return=Region_lookup(region) hrus = Region_return[0] nhru = hrus ScienceBase_URL= Region_return[1] pyGDP.WFS_URL = ScienceBase_URL # call web processing module pyGDP = pyGDP.pyGDPwebProcessing() # change working directory so the GDP output will be written there # Datasets and their properties # run 1, 2 , and 3 #********************************** # run 1 only #scenario = 'a2' # other scenarios are 'a1b' and 'b1' scenarios = ['a2','a1b'] # not running b1 or a1fi #ends 2012 timestart = '1950-01-15T00:00:00.000Z' timeend='1960-12-15T00:00:00.000Z' shapefiles = pyGDP.getShapefiles() for shp in shapefiles: print shp shapefile = 'sb:nhru' user_attribute = 'hru_id_loc' user_value = None #os.chdir('C:\\Users\\reimandy\\workspace\\userpackages\\WaterBalanceModel\\Step1_CLIMATE_DATA\\'+region+'\\MAURERBREKE') dir = currdir gcmRun = '1' for scenario in scenarios: cccma_cgcm3_1_1 = ['sres'+scenario+'_cccma-cgcm3-1_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_cccma-cgcm3-1_1_Prcp', 'sres'+scenario+'_cccma-cgcm3-1_1_Tavg'] gfdl_cm2_1_1_1 = ['sres'+scenario+'_gfdl-cm2-1_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_gfdl-cm2-1_1_Prcp', 'sres'+scenario+'_gfdl-cm2-1_1_Tavg'] miroc3_2_medres_1 = ['sres'+scenario+'_miroc3-2-medres_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_miroc3-2-medres_1_Prcp', 'sres'+scenario+'_miroc3-2-medres_1_Tavg'] miub_echo_g_1_1 = ['sres'+scenario+'_miub-echo-g_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_miub-echo-g_1_Prcp', 'sres'+scenario+'_miub-echo-g_1_Tavg'] mpi_echam5_1 = ['sres'+scenario+'_mpi-echam5_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_mpi-echam5_1_Prcp', 'sres'+scenario+'_mpi-echam5_1_Tavg'] mri_cgcm2_3_2a_1 = ['sres'+scenario+'_mri-cgcm2-3-2a_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_mri-cgcm2-3-2a_1_Prcp', 'sres'+scenario+'_mri-cgcm2-3-2a_1_Tavg'] # New MaurerBreke Statistically downscaled datasets (put with other MB datasets) bccr_bcm2_0 = ['sres'+scenario+'_bccr-bcm2-0_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_bccr-bcm2-0_'+gcmRun+'_Prcp', 'sres'+scenario+'_bccr-bcm2-0_'+gcmRun+'_Tavg'] cnrm_cm3 = ['sres'+scenario+'_cnrm-cm3_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_cnrm-cm3_'+gcmRun+'_Prcp', 'sres'+scenario+'_cnrm-cm3_'+gcmRun+'_Tavg'] csiro_mk3_0 = ['sres'+scenario+'_csiro-mk3-0_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_csiro-mk3-0_'+gcmRun+'_Prcp', 'sres'+scenario+'_csiro-mk3-0_'+gcmRun+'_Tavg'] giss_model_e_r = ['sres'+scenario+'_giss-model-e-r_2','dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_giss-model-e-r_2_Prcp', 'sres'+scenario+'_giss-model-e-r_2_Tavg'] inmcm3_0 = ['sres'+scenario+'_inmcm3-0_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_inmcm3-0_'+gcmRun+'_Prcp', 'sres'+scenario+'_inmcm3-0_'+gcmRun+'_Tavg'] ipsl_cm4 = ['sres'+scenario+'_ipsl-cm4_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_ipsl-cm4_'+gcmRun+'_Prcp', 'sres'+scenario+'_ipsl-cm4_'+gcmRun+'_Tavg'] ncar_ccsm3_0 = ['sres'+scenario+'_ncar-ccsm3-0_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_ncar-ccsm3-0_'+gcmRun+'_Prcp', 'sres'+scenario+'_ncar-ccsm3-0_'+gcmRun+'_Tavg'] ncar_pcm1 = ['sres'+scenario+'_ncar-pcm1_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_ncar-pcm1_'+gcmRun+'_Prcp', 'sres'+scenario+'_ncar-pcm1_'+gcmRun+'_Tavg'] ukmo_hadcm3 = ['sres'+scenario+'_ukmo-hadcm3_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\ 'sres'+scenario+'_ukmo-hadcm3_'+gcmRun+'_Prcp', 'sres'+scenario+'_ukmo-hadcm3_'+gcmRun+'_Tavg'] data = [cccma_cgcm3_1_1,gfdl_cm2_1_1_1,miroc3_2_medres_1,miub_echo_g_1_1,mpi_echam5_1,mri_cgcm2_3_2a_1] for dataset in data: if len(scenario) == 2: name = dataset[0] name = name[7:] else: name = dataset[0] name = name[8:] file_loc = str(dir.name)+'\\Step1_CLIMATE_DATA\\'+region+'\\'+scenario+'\\'+name if not os.path.exists(file_loc): os.mkdir(file_loc) os.chdir(file_loc) print "The current dataset being worked on is: " + name dataSet = dataset[1] dataType = dataset[5:] timestep = dataset[2] timeBegin = dataset[3] timeEnd = dataset[4] gmlIDs = None verbose = True coverage = Region_return[2] delim = 'COMMA' stats = 'MEAN' start = time.time() outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats) end = time.time() print "Start time is: " + str(start) print 'End time is: ' + str(end) print 'Total time was: ' + str(end-start) print outputPath shutil.copy2(outputPath, region+'_'+name+'.csv') csvfile = os.getcwd()+region+'_'+name+'.csv' # dataType = ['Prcp', 'Tavg'] # vistrails_data_set = ['Prcp', 'Tavg'] #csvread = csv.reader(open(region+'_'+name+'.csv', 'rb')) #csvwrite = csv.writer(open(dataType[0]+'.csv', 'wb')) #parsedFiles.append(dataType[0]+'.csv') #index = 0 #temp = csvread #var = temp.next() #var[0] = '#'+dataType[0] #gage = temp.next() #csvwrite.writerow(var) #csvwrite.writerow(gage) # for variable in dataType: # # for row in csvread: # if variable == dataType[len(dataType) - 1]: # csvwrite.writerow(row) # else: # if (row[0] in '#'+dataType[index+1]) or (row[0] in '# ' + dataType[index+1]): # var = '#'+dataType[index+1] # csvwrite = csv.writer(open(dataType[index+1] + '.csv', 'wb')) # parsedFiles.append(dataType[index+1]+'.csv') # row[1:] = '' # row[0] = var # csvwrite.writerow(row) # csvwrite.writerow(gage) # # if len(dataType) == 2: # csvwrite.writerow(csvread.next()) # else: # csvread.next() # csvwrite.writerow(csvread.next()) # csvwrite.writerow(csvread.next()) # break # else: # if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]: # csvwrite.writerow(row) # print 'Finished parsing ' + variable + '.csv' # # if (index+1)<len(dataType): # index += 1 # # os.chdir(dir) #main_func('nhru', os.getcwd(), '', '')
baseURI = 'http://cida.usgs.gov/thredds/dodsC/loca_future' startTime = 1950 endTime = 2100 varID = 'pr_CCSM4_r6i1p1_rcp45' #['tasmax_CCSM4_r6i1p1_rcp45', 'tasmin_CCSM4_r6i1p1_rcp45'] years = array.array('i', (i for i in range(startTime, endTime + 1))) fileList = [] for yr in years: datasetURI = '%s%s%s' % (baseURI, yr, '.nc') timeRange = pyGDP.getTimeRange(datasetURI, varID) timeStart = timeRange[0] timeEnd = timeRange[1] print "process beginning for %s on year %s'" % (varID, yr) outputFile_handle = pyGDP.submitFeatureWeightedGridStatistics( self, geoType, dataSetURI, varID, startTime, endTime, '', '', '', '', coverage='true', delim='COMMA') fileList.append(outputFile_handle)
# Note that removing "value" in the main request below will run all values in the shapefile. value = ['Delaware'] # Search for datasets dataSetURIs = pyGDP.getDataSetURI(anyText='wicci') for dataset in dataSetURIs: print(dataset) # Loop through datasets of interest, in this case the first three OPeNDAP urls. for dataSetURI in dataSetURIs[1][2][0:3]: # Get the available data types associated with the dataSetURI dataTypes = pyGDP.getDataType(dataSetURI) print(dataTypes) # For this example just run the first dataType. This dataType list should be modified if multiple # datatypes are required. dataType = dataTypes[0] print(dataType) # Get available time range for the dataset. timeRange = pyGDP.getTimeRange(dataSetURI, dataType) print(timeRange) # Execute a GeatureWeightedGridStatistics request and return the path to the output file. # Note that this is for one time step but could be for multiple. Please test on very short # time periods to minimize impacts on system resources. outputfile = pyGDP.submitFeatureWeightedGridStatistics( shapefile, dataSetURI, dataType, timeRange[0], timeRange[0], usr_attribute, value) print(outputfile)
import pyGDP pyGDP = pyGDP.pyGDPwebProcessing() """ This example shows how to use multiple dataTypes and Statistics. """ shapefile = 'sample:simplified_huc8' user_attribute = 'SUBBASIN' user_value = 'Baraboo' dataSet = 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml' dataType = ['Prcp', 'Tavg', 'Tmax', 'Tmin'] timeBegin = '1970-01-24T00:00:00.000Z' timeEnd = '1970-01-25T00:00:00.000Z' gmlIDs = None verbose = True coverage = 'true' delim = 'COMMA' stats = ['MEAN', 'STD_DEV'] print 'Processing request.' outputPath = pyGDP.submitFeatureWeightedGridStatistics( shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats)
from __future__ import print_function import pyGDP pyGDP = pyGDP.pyGDPwebProcessing() """ This example shows how to use multiple dataTypes and Statistics. """ shapefile = 'sample:simplified_huc8' user_attribute = 'SUBBASIN' user_value = 'Baraboo' dataSet = 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml' dataType = ['Prcp', 'Tavg', 'Tmax', 'Tmin'] timeBegin = '1970-01-24T00:00:00.000Z' timeEnd = '1970-01-25T00:00:00.000Z' gmlIDs = None verbose = True coverage = 'true' delim = 'COMMA' stats = ['MEAN', 'STD_DEV'] print('Processing request.') outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats)
) + '_' + dataURI.replace( 'http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/ncml/', '') outputfilename = outputfilename.replace( 'http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/averages_rcp/ncml/', '') outputfilename = outputfilename.replace( 'http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/averages_hist/ncml/', '') outputfilename = outputfilename.replace( 'http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/averages_gmo/ncml/', '') outputfilename = outputfilename.replace( 'http://localhost:8080/thredds/dodsC/Scratch/thredds/bcca/bcca/cmip5/derivatives/cmip5_obs_der/', '') if not os.path.isfile(outputfilename): open(outputfilename, 'a').close() print shapefile print dataURI print dataTypes print timeRange print shapefiles[shapefile] outFile = pyGDP.submitFeatureWeightedGridStatistics( geoType=shapefile, dataSetURI=dataURI, varID=dataTypes, startTime=timeRange[0], endTime=timeRange[1], attribute=shapefiles[shapefile], coverage=False, outputfname=outputfilename)