def main_func(region, currdir, timestart, timeend):#, scenarios):
def Region_lookup(region):
region_properties = []
if region == 'nhru':
region_properties.append(371)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d35da1e4b0ca184833940c')
region_properties.append('false')
elif region == 'R01':
region_properties.append(2462)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5244735ae4b05b217bada04e')
region_properties.append('false')
elif region == 'R02':
region_properties.append(4827)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52696784e4b0584cbe9168ee')
region_properties.append('false')
elif region == 'R03':
region_properties.append(9899)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5283bd23e4b047efbbb57922')
region_properties.append('false')
elif region == 'R04':
region_properties.append(5936)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5284ff57e4b063f258e61b9d')
region_properties.append('false')
elif region == 'R05':
region_properties.append(7182)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/528516bbe4b063f258e62161')
region_properties.append('false')
elif region == 'R06':
region_properties.append(2303)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d75296e4b055e0afd5be2c')
region_properties.append('true')
elif region == 'R07':
region_properties.append(8205)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52851cd5e4b063f258e643dd')
region_properties.append('true')
elif region == 'R08':
region_properties.append(4449)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52854695e4b063f258e6513c')
region_properties.append('true')
elif region == 'R10L':
region_properties.append(8603)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/520031dee4b0ad2d97189db2')
region_properties.append('true')
elif region =='R10U':
region_properties.append(10299)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5216849ee4b0b45d6ba61e2e')
region_properties.append('false')
elif region == 'R11':
region_properties.append(7373)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d1f9ebe4b08b18a62d586b')
region_properties.append('true')
elif region == 'R12':
region_properties.append(7815)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5201328ae4b009d47a4c247a')
region_properties.append('false')
elif region == 'R13':
region_properties.append(1958)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d752b9e4b055e0afd5be36')
region_properties.append('false')
elif region == 'R14':
region_properties.append(3879)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52029c68e4b0e21cafa4b40c')
region_properties.append('false')
elif region == 'R15':
region_properties.append(3441)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5285389ae4b063f258e64863')
region_properties.append('false')
elif region == 'R16':
region_properties.append(2664)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52853f97e4b063f258e64875')
region_properties.append('false')
return region_properties
global parsedFiles, csvfile, vistrails_data_set, nhru, url
import pyGDP
#region = 'R13'
parsedFiles = []
vistrails_data_set = []
Region_return=Region_lookup(region)
hrus = Region_return[0]
nhru = hrus
ScienceBase_URL= Region_return[1]
pyGDP.WFS_URL = ScienceBase_URL
# call web processing module
pyGDP = pyGDP.pyGDPwebProcessing()
# change working directory so the GDP output will be written there
# Datasets and their properties
# run 1, 2 , and 3
#**********************************
# run 1 only
#scenario = 'a2'
# other scenarios are 'a1b' and 'b1'
scenarios = ['a2','a1b'] # not running b1 or a1fi
#ends 2012
timestart = '1950-01-15T00:00:00.000Z'
timeend='1960-12-15T00:00:00.000Z'
shapefiles = pyGDP.getShapefiles()
for shp in shapefiles:
print shp
shapefile = 'sb:nhru'
user_attribute = 'hru_id_loc'
user_value = None
#os.chdir('C:\\Users\\reimandy\\workspace\\userpackages\\WaterBalanceModel\\Step1_CLIMATE_DATA\\'+region+'\\MAURERBREKE')
dir = currdir
gcmRun = '1'
for scenario in scenarios:
cccma_cgcm3_1_1 = ['sres'+scenario+'_cccma-cgcm3-1_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_cccma-cgcm3-1_1_Prcp',
'sres'+scenario+'_cccma-cgcm3-1_1_Tavg']
gfdl_cm2_1_1_1 = ['sres'+scenario+'_gfdl-cm2-1_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_gfdl-cm2-1_1_Prcp',
'sres'+scenario+'_gfdl-cm2-1_1_Tavg']
miroc3_2_medres_1 = ['sres'+scenario+'_miroc3-2-medres_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_miroc3-2-medres_1_Prcp',
'sres'+scenario+'_miroc3-2-medres_1_Tavg']
miub_echo_g_1_1 = ['sres'+scenario+'_miub-echo-g_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_miub-echo-g_1_Prcp',
'sres'+scenario+'_miub-echo-g_1_Tavg']
mpi_echam5_1 = ['sres'+scenario+'_mpi-echam5_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_mpi-echam5_1_Prcp',
'sres'+scenario+'_mpi-echam5_1_Tavg']
mri_cgcm2_3_2a_1 = ['sres'+scenario+'_mri-cgcm2-3-2a_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_mri-cgcm2-3-2a_1_Prcp',
'sres'+scenario+'_mri-cgcm2-3-2a_1_Tavg']
# New MaurerBreke Statistically downscaled datasets (put with other MB datasets)
bccr_bcm2_0 = ['sres'+scenario+'_bccr-bcm2-0_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_bccr-bcm2-0_'+gcmRun+'_Prcp',
'sres'+scenario+'_bccr-bcm2-0_'+gcmRun+'_Tavg']
cnrm_cm3 = ['sres'+scenario+'_cnrm-cm3_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_cnrm-cm3_'+gcmRun+'_Prcp',
'sres'+scenario+'_cnrm-cm3_'+gcmRun+'_Tavg']
csiro_mk3_0 = ['sres'+scenario+'_csiro-mk3-0_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_csiro-mk3-0_'+gcmRun+'_Prcp',
'sres'+scenario+'_csiro-mk3-0_'+gcmRun+'_Tavg']
giss_model_e_r = ['sres'+scenario+'_giss-model-e-r_2','dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_giss-model-e-r_2_Prcp',
'sres'+scenario+'_giss-model-e-r_2_Tavg']
inmcm3_0 = ['sres'+scenario+'_inmcm3-0_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_inmcm3-0_'+gcmRun+'_Prcp',
'sres'+scenario+'_inmcm3-0_'+gcmRun+'_Tavg']
ipsl_cm4 = ['sres'+scenario+'_ipsl-cm4_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_ipsl-cm4_'+gcmRun+'_Prcp',
'sres'+scenario+'_ipsl-cm4_'+gcmRun+'_Tavg']
ncar_ccsm3_0 = ['sres'+scenario+'_ncar-ccsm3-0_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_ncar-ccsm3-0_'+gcmRun+'_Prcp',
'sres'+scenario+'_ncar-ccsm3-0_'+gcmRun+'_Tavg']
ncar_pcm1 = ['sres'+scenario+'_ncar-pcm1_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_ncar-pcm1_'+gcmRun+'_Prcp',
'sres'+scenario+'_ncar-pcm1_'+gcmRun+'_Tavg']
ukmo_hadcm3 = ['sres'+scenario+'_ukmo-hadcm3_'+gcmRun,'dods://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml','Monthly',timestart,timeend,\
'sres'+scenario+'_ukmo-hadcm3_'+gcmRun+'_Prcp',
'sres'+scenario+'_ukmo-hadcm3_'+gcmRun+'_Tavg']
data = [cccma_cgcm3_1_1,gfdl_cm2_1_1_1,miroc3_2_medres_1,miub_echo_g_1_1,mpi_echam5_1,mri_cgcm2_3_2a_1]
for dataset in data:
if len(scenario) == 2:
name = dataset[0]
name = name[7:]
else:
name = dataset[0]
name = name[8:]
file_loc = str(dir.name)+'\\Step1_CLIMATE_DATA\\'+region+'\\'+scenario+'\\'+name
if not os.path.exists(file_loc):
os.mkdir(file_loc)
os.chdir(file_loc)
print "The current dataset being worked on is: " + name
dataSet = dataset[1]
dataType = dataset[5:]
timestep = dataset[2]
timeBegin = dataset[3]
timeEnd = dataset[4]
gmlIDs = None
verbose = True
coverage = Region_return[2]
delim = 'COMMA'
stats = 'MEAN'
start = time.time()
outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats)
end = time.time()
print "Start time is: " + str(start)
print 'End time is: ' + str(end)
print 'Total time was: ' + str(end-start)
print outputPath
shutil.copy2(outputPath, region+'_'+name+'.csv')
csvfile = os.getcwd()+region+'_'+name+'.csv'
# dataType = ['Prcp', 'Tavg']
# vistrails_data_set = ['Prcp', 'Tavg']
#csvread = csv.reader(open(region+'_'+name+'.csv', 'rb'))
#csvwrite = csv.writer(open(dataType[0]+'.csv', 'wb'))
#parsedFiles.append(dataType[0]+'.csv')
#index = 0
#temp = csvread
#var = temp.next()
#var[0] = '#'+dataType[0]
#gage = temp.next()
#csvwrite.writerow(var)
#csvwrite.writerow(gage)
# for variable in dataType:
#
# for row in csvread:
# if variable == dataType[len(dataType) - 1]:
# csvwrite.writerow(row)
# else:
# if (row[0] in '#'+dataType[index+1]) or (row[0] in '# ' + dataType[index+1]):
# var = '#'+dataType[index+1]
# csvwrite = csv.writer(open(dataType[index+1] + '.csv', 'wb'))
# parsedFiles.append(dataType[index+1]+'.csv')
# row[1:] = ''
# row[0] = var
# csvwrite.writerow(row)
# csvwrite.writerow(gage)
#
# if len(dataType) == 2:
# csvwrite.writerow(csvread.next())
# else:
# csvread.next()
# csvwrite.writerow(csvread.next())
# csvwrite.writerow(csvread.next())
# break
# else:
# if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]:
# csvwrite.writerow(row)
# print 'Finished parsing ' + variable + '.csv'
#
# if (index+1)<len(dataType):
# index += 1
#
# os.chdir(dir)
#main_func('nhru', os.getcwd(), '', '')
from __future__ import print_function
import pyGDP
'''
Shows GDP workflow in using specific values of attributes from a shapefile
that already exists on the GDP server.
'''
pyGDP = pyGDP.pyGDPwebProcessing()
shapefiles = pyGDP.getShapefiles()
print('Available shapefiles: ')
for shapefile in shapefiles:
print(shapefile)
# Grab the file and get its attributes:
shapefile = 'sample:CSC_Boundaries'
attributes = pyGDP.getAttributes(shapefile)
for attr in attributes:
print(attr)
# Grab the values from 'area_name' and 'sample:CSC_Boundaries'
usr_attribute = 'area_name'
values = pyGDP.getValues(shapefile, usr_attribute)
for v in values:
print(v)
usr_value = 'Southwest'
dataSetURI = 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml'
dataTypes = pyGDP.getDataType(dataSetURI)
for d in dataTypes:
print(d)
def main_func(curdir, set, region):
def Region_lookup(region):
region_properties = []
if region == 'nhru':
region_properties.append(371)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d35da1e4b0ca184833940c')
region_properties.append('false')
if region == 'R01':
region_properties.append(2462)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5244735ae4b05b217bada04e')
region_properties.append('false')
if region == 'R02':
region_properties.append(4827)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52696784e4b0584cbe9168ee')
region_properties.append('false')
if region == 'R03':
region_properties.append(9899)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5283bd23e4b047efbbb57922')
region_properties.append('false')
if region == 'R04':
region_properties.append(5936)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5284ff57e4b063f258e61b9d')
region_properties.append('false')
if region == 'R05':
region_properties.append(7182)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/528516bbe4b063f258e62161')
region_properties.append('false')
if region == 'R06':
region_properties.append(2303)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d75296e4b055e0afd5be2c')
region_properties.append('true')
if region == 'R07':
region_properties.append(8205)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52851cd5e4b063f258e643dd')
region_properties.append('true')
if region == 'R08':
region_properties.append(4449)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52854695e4b063f258e6513c')
region_properties.append('true')
if region == 'R10L':
region_properties.append(8603)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/520031dee4b0ad2d97189db2')
region_properties.append('true')
elif region =='R10U':
region_properties.append(10299)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5216849ee4b0b45d6ba61e2e')
region_properties.append('false')
elif region == 'R11':
region_properties.append(7373)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d1f9ebe4b08b18a62d586b')
region_properties.append('true')
elif region == 'R12':
region_properties.append(7815)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5201328ae4b009d47a4c247a')
region_properties.append('false')
elif region == 'R13':
region_properties.append(1958)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d752b9e4b055e0afd5be36')
region_properties.append('false')
elif region == 'R14':
region_properties.append(3879)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52029c68e4b0e21cafa4b40c')
region_properties.append('false')
elif region == 'R15':
region_properties.append(3441)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5285389ae4b063f258e64863')
region_properties.append('false')
elif region == 'R16':
region_properties.append(2664)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52853f97e4b063f258e64875')
region_properties.append('false')
return region_properties
def list_define(set):
if set == 'cccma_cgmc3_1':
return ['cccma_cgmc3'+str(scenario), 'http://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
elif set == 'cnrm_cm3':
return ['cnrm_cm3'+str(scenario), 'http://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
elif set == 'gfdl_cm2_1':
return ['gfdl_cm2_1'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
elif set == 'ipsl_cm4':
return ['ipsl_cm4'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
elif set == 'miroc3_2_medres':
return ['miroc3_2_medres'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
elif set == 'miub_echo_g':
return ['miub_echo_g'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
elif set == 'mri_cgcm2_3_2a':
return ['mri_cgcm2_3_2a'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
elif set == 'mpi_echam5':
return ['mpi_echam5'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
global csvfile, vt_dataset, nhru, length, vt_datatype, url
import pyGDP
Region_return=Region_lookup(region)
hrus = Region_return[0]
nhru = hrus
ScienceBase_URL= Region_return[1]
pyGDP.WFS_URL = ScienceBase_URL
url = pyGDP.WFS_URL
pyGDP = pyGDP.pyGDPwebProcessing()
# change working directory so the GDP output will be written there
# Datasets and their properties
# run 1, 2 , and 3
#**********************************
# run 1 only
#scenario = 'a2'
# other scenarios are 'a1b' and 'b1'
#scenarios = ['a2','a1b','b1']
scenarios = ['a2', 'a1b']
timestart = '2046-01-15T00:00:00.000Z'
timeend='2100-12-15T00:00:00.000Z'
#**********************************
# Datasets for each scenario - note that mpi_echam5 is not run for scenario a1b
#a1b
#data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mri_cgcm2_3_2a]
#a2
#data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mpi_echam5,mri_cgcm2_3_2a]
#b1
#data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mpi_echam5,mri_cgcm2_3_2a]
# get list of shapefiles uploaded to the GDP
shapefiles = pyGDP.getShapefiles()
for shp in shapefiles:
print shp
# feature loaded from sciencebase
#should shapefile be sb:SP_hru instead?
shapefile = 'sb:nhru'
user_attribute = 'hru_id_loc'
user_value = None
os.chdir(curdir)
dir = os.getcwd()
vt_data = list_define(set)
vt_datatype = vt_data[5:]
#for scenario in scenarios:
# cnrm_cm3 = ['cnrm_cm3'+str(scenario), 'http://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
# 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
# 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
# 'cnrm_cm3-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
#
# gfdl_cm2_1 = ['gfdl_cm2_1'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
# 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
# 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
# 'gfdl_cm2_1-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
#
# ipsl_cm4 = ['ipsl_cm4'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
# 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
# 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
# 'ipsl_cm4-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
#
# mpi_echam5 = ['mpi_echam5'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
# 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
# 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
# 'mpi_echam5-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
#
# miroc3_2_medres = ['miroc3_2_medres'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
# 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
# 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
# 'miroc3_2_medres-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
#
# cccma_cgmc3_1 = ['cccma_cgmc3'+str(scenario), 'http://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
# 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
# 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
# 'cccma_cgcm3_1-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
#
# miub_echo_g = ['miub_echo_g'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
# 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
# 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
# 'miub_echo_g-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
#
# mri_cgcm2_3_2a = ['mri_cgcm2_3_2a'+str(scenario), 'dods://pcmdi8.llnl.gov/thredds/dodsC/bcca/BCCA_0.125deg-gregorian-sres-monthly', 'Monthly', timestart, timeend,\
# 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-pr-BCCA_0-125deg-monthly',
# 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-tasmax-BCCA_0-125deg-monthly',
# 'mri_cgcm2_3_2a-gregorian-sres'+str(scenario)+'-run1-tasmin-BCCA_0-125deg-monthly']
# if scenario == 'a1b':
# data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mri_cgcm2_3_2a]
# elif scenario == 'a2':
# data=[cccma_cgmc3_1,cnrm_cm3,gfdl_cm2_1,ipsl_cm4,miroc3_2_medres,miub_echo_g,mpi_echam5,mri_cgcm2_3_2a]
timestart = time.time()
#################
#scenarios = ['a2', 'a1b']
scenario = 'a2'
#for dataset in data:
#file_loc = dir+'\\'+scenario+'\\'+dataset[0]
file_loc = dir+'\\'+set
if not os.path.exists(file_loc):
os.mkdir(file_loc)
os.chdir(file_loc)
#print "The current scenario is: " +scenario + "dataset being worked on is: " + dataset[0]
print "The current scenario is: " +scenario + "dataset being worked on is: " + set
#The url of each dataset
dataSet = vt_data[1]
#dataSet = dataset[1]
#The precipitation and temperatures of each dataset
#Start at position(not index) 5 until the end of the
#dictionary's key(which is a list)
dataType=vt_data[5:]
#dataType = dataset[5:]
# daily or monthly for additional aggregation/formatting (not appended yet)
timeStep = vt_data[2]
length = timeStep
#timeStep = dataset[2]
#Start date
timeBegin = vt_data[3]
#timeBegin = dataset[3]
#End date
timeEnd = vt_data[4]
#timeEnd = dataset[4]
# data processing arguments
gmlIDs=None
verbose=True
#coverage = 'false' check if on US border/ocean
coverage = Region_return[2]
delim='COMMA'
stats='MEAN'
# run the pyGDP
start = time.time()
outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats)
end = time.time()
print 'Start time is: ' + str(start)
print 'End time is: ' + str(end)
print 'Total time was: ' + str(end-start)
print outputPath
# copy the output and rename it
# ind = 5
# for var in range(5, len(dataset)):
# line = dataset[var].split('-')
# dataset[ind] = line[4]
# ind += 1
#
# dataType = dataset[5:]
#shutil.copy2(outputPath, region+'_'+dataset[0]+'.csv')
shutil.copy2(outputPath, region+'_'+vt_data[0]+'.csv')
# csvfile = os.getcwd()+region+'_'+dataset[0]+'.csv'
# #Parse the csv file
# index = 0
#
# csvread = csv.reader(open(region+'_'+dataset[0] + '.csv', "rb"))
#
# csvwrite = csv.writer(open(dataType[0] + '.csv', "wb"))
#
# temp = csvread
# var = temp.next()
# var[0] = '#'+dataType[0]
# #Gets gage ids
# gage = temp.next()
#
# #Writes current variable to csv file
# csvwrite.writerow(var)
# #Writes all gage ids to csv file
# csvwrite.writerow(gage)
#
# for variable in dataType:
#
# for row in csvread:
# #if on last variable
# if variable == dataType[len(dataType) - 1]:
# csvwrite.writerow(row)
# else:
# if (row[0] in '#'+dataType[index+1]) or (row[0] in '# '+dataType[index+1]):
# #Line 33 is used for titling the csv file the name of the variable (like tmin, ppt, or tmax)
# var = '#'+dataType[index+1]
# parsedFiles.append(os.getcwd()+'\\'+variable+'.csv')
# csvwrite = csv.writer(open(dataType[index+1] + '.csv', "wb"))
# row[1:] = ""
# row[0] = var
# csvwrite.writerow(row)
# csvwrite.writerow(gage)
# if len(dataType) == 2:
# csvread.next()
# else:
# csvread.next()
# csvwrite.writerow(csvread.next())
# csvwrite.writerow(csvread.next())
# break
# else:
# if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]:
# csvwrite.writerow(row)
# print "Finished parsing " + variable + ".csv"
# parsedFiles.append(os.getcwd()+'\\'+variable+'.csv')
# # use index to keep track of next variable
# if (index + 1) < len(dataType):
# index += 1
timeend = time.time()
print 'Start time of pyGDP: ' + str(timestart)
print 'End time of pyGDP: ' + str(timeend)
print 'Total time of pyGDP: ' + str(timeend-timestart)
os.chdir(dir)
import pyGDP
pyGDP = pyGDP.pyGDPwebProcessing()
"""
This example script calls into the geoserver to obtain
the name of the shapefile 'sample:CONUS_States' sets up the
proper inputs and submits a request into GDP.
"""
shapefiles = pyGDP.getShapefiles()
print 'Available shapefiles: '
for shapefile in shapefiles:
print shapefile
print
# Grab the file and get its attributes:
shapefile = 'sample:CONUS_States'
attributes = pyGDP.getAttributes(shapefile)
for attr in attributes:
print attr
print
# Grab the values from 'OBJECTID' and 'upload:OKCNTYD'
usr_attribute = 'STATE'
values = pyGDP.getValues(shapefile, usr_attribute)
for v in values:
print v
print
# our shapefile = 'upload:OKCNTYD', usr_attribute = 'OBJECTID', and usr_value = 13
# We get the dataset URI that we are interested in
def main_func():
def Region_lookup(region):
region_properties = []
if region == 'nhru':
region_properties.append(371)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d35da1e4b0ca184833940c')
region_properties.append('false')
if region == 'R01':
region_properties.append(2462)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5244735ae4b05b217bada04e')
region_properties.append('false')
if region == 'R02':
region_properties.append(4827)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52696784e4b0584cbe9168ee')
region_properties.append('false')
if region == 'R03':
region_properties.append(9899)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5283bd23e4b047efbbb57922')
region_properties.append('false')
if region == 'R04':
region_properties.append(5936)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5284ff57e4b063f258e61b9d')
region_properties.append('false')
if region == 'R05':
region_properties.append(7182)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/528516bbe4b063f258e62161')
region_properties.append('false')
if region == 'R06':
region_properties.append(2303)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d75296e4b055e0afd5be2c')
region_properties.append('true')
if region == 'R07':
region_properties.append(8205)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52851cd5e4b063f258e643dd')
region_properties.append('true')
if region == 'R08':
region_properties.append(4449)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52854695e4b063f258e6513c')
region_properties.append('true')
if region == 'R10L':
region_properties.append(8603)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/520031dee4b0ad2d97189db2')
region_properties.append('true')
elif region =='R10U':
region_properties.append(10299)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5216849ee4b0b45d6ba61e2e')
region_properties.append('false')
elif region == 'R11':
region_properties.append(7373)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d1f9ebe4b08b18a62d586b')
region_properties.append('true')
elif region == 'R12':
region_properties.append(7815)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5201328ae4b009d47a4c247a')
region_properties.append('false')
elif region == 'R13':
region_properties.append(1958)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/51d752b9e4b055e0afd5be36')
region_properties.append('false')
elif region == 'R14':
region_properties.append(3879)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52029c68e4b0e21cafa4b40c')
region_properties.append('false')
elif region == 'R15':
region_properties.append(3441)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/5285389ae4b063f258e64863')
region_properties.append('false')
elif region == 'R16':
region_properties.append(2664)
region_properties.append('https://www.sciencebase.gov/catalogMaps/mapping/ows/52853f97e4b063f258e64875')
region_properties.append('false')
return region_properties
global parsedFiles, csvfile, vistrails_data_set, nhru, length
regions = ['R06', 'R10L', 'R11', 'R12', 'R13', 'R14']
region = 'R06'
parsedFiles = []
csvfile = ''
vistrails_data_set = []
Region_return=Region_lookup(region)
hrus = Region_return[0]
nhru = hrus
ScienceBase_URL= Region_return[1]
pyGDP.WFS_URL = ScienceBase_URL
# call web processing module
pyGDP = pyGDP.pyGDPwebProcessing()
# change working directory so the GDP output will be written there
# Datasets and their properties
# run 1, 2 , and 3
#**********************************
# run 1 only
#scenario = 'a2'
# other scenarios are 'a1b' and 'b1'
scenarios = ['a2','a1b'] # not running b1 or a1fi
timestart = '1960-01-15T00:00:00.000Z'
timeend = '2010-12-15T00:00:00.000Z'
# timestart = '2010-12-15T00:00:00.000Z'
# timeend='2099-12-15T00:00:00.000Z'
#**********************************
# datasets greyed out - we are not initially using.
##ccsm = ['ccsm'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
## 'ccsm-'+scenarios[0]+'-pr-NAm-grid',
## 'ccsm-'+scenarios[0]+'-tmax-NAm-grid',
## 'ccsm-'+scenarios[0]+'-tmin-NAm-grid']
shapefiles = pyGDP.getShapefiles()
for shp in shapefiles:
print shp
# feature loaded from sciencebase
#should shapefile be sb:SP_hru instead?
shapefile = 'sb:'+region+'_hru'
user_attribute = 'hru_id_loc'
user_value = None
os.chdir('C:\\Users\\reimandy\\workspace\\userpackages\\WaterBalanceModel\\Step1_CLIMATE_DATA\\'+region+'\\HAYHOE')
dir = os.getcwd()
for scenario in scenarios:
cgcm3_t47 = ['cgcm3_t47'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
'cgcm3_t47-'+str(scenario)+'-pr-NAm-grid',
'cgcm3_t47-'+str(scenario)+'-tmax-NAm-grid',
'cgcm3_t47-'+str(scenario)+'-tmin-NAm-grid']
cgcm3_t63 = ['cgcm3_t63'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
'cgcm3_t63-'+str(scenario)+'-pr-NAm-grid',
'cgcm3_t63-'+str(scenario)+'-tmax-NAm-grid',
'cgcm3_t63-'+str(scenario)+'-tmin-NAm-grid']
cnrm = ['cnrm'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
'cnrm-'+str(scenario)+'-pr-NAm-grid',
'cnrm-'+str(scenario)+'-tmax-NAm-grid',
'cnrm-'+str(scenario)+'-tmin-NAm-grid']
##csiro = ['csiro'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
## 'csiro-'+scenarios[0]+'-pr-NAm-grid',
## 'csiro-'+scenarios[0]+'-tmax-NAm-grid',
## 'csiro-'+scenarios[0]+'-tmin-NAm-grid']
##echam5 = ['echam5'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
## 'echam5-'+scenarios[0]+'-pr-NAm-grid',
## 'echam5-'+scenarios[0]+'-tmax-NAm-grid',
## 'echam5-'+scenarios[0]+'-tmin-NAm-grid']
echo = ['echo'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
'echo-'+str(scenario)+'-pr-NAm-grid',
'echo-'+str(scenario)+'-tmax-NAm-grid',
'echo-'+str(scenario)+'-tmin-NAm-grid']
gfdl_2_1 = ['gfdl_2-1'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
'gfdl_2-1-'+str(scenario)+'-pr-NAm-grid',
'gfdl_2-1-'+str(scenario)+'-tmax-NAm-grid',
'gfdl_2-1-'+str(scenario)+'-tmin-NAm-grid']
##giss_aom = ['giss_aom'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
## 'giss_aom-'+scenarios[0]+'-pr-NAm-grid',
## 'giss_aom-'+scenarios[0]+'-tmax-NAm-grid',
## 'giss_aom-'+scenarios[0]+'-tmin-NAm-grid']
##hadcm3 = ['hadcm3'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
## 'hadcm3-'+scenarios[0]+'-pr-NAm-grid',
## 'hadcm3-'+scenarios[0]+'-tmax-NAm-grid',
## 'hadcm3-'+scenarios[0]+'-tmin-NAm-grid']
##miroc_hi = ['miroc_hi'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
## 'miroc_hi-'+scenarios[0]+'-pr-NAm-grid',
## 'miroc_hi-'+scenarios[0]+'-tmax-NAm-grid',
## 'miroc_hi-'+scenarios[0]+'-tmin-NAm-grid']
miroc_med = ['miroc_med'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
'miroc_med-'+str(scenario)+'-pr-NAm-grid',
'miroc_med-'+str(scenario)+'-tmax-NAm-grid',
'miroc_med-'+str(scenario)+'-tmin-NAm-grid']
mri_cgcm2 = ['mri_cgcm2'+str(scenario), 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
'mri_cgcm2-'+str(scenario)+'-pr-NAm-grid',
'mri_cgcm2-'+str(scenario)+'-tmax-NAm-grid',
'mri_cgcm2-'+str(scenario)+'-tmin-NAm-grid']
##pcm = ['pcm'+scenarios[0], 'http://cida.usgs.gov/thredds/dodsC/dcp/conus', 'Daily', timestart, timeend,\
## 'pcm-'+scenarios[0]+'-pr-NAm-grid',
## 'pcm-'+scenarios[0]+'-tmax-NAm-grid',
## 'pcm-'+scenarios[0]+'-tmin-NAm-grid']
# get list of shapefiles uploaded to the GDP
if scenario == 'a1b':
#data=[cgcm3_t47,cgcm3_t63,cnrm,echam5,echo,giss_aom,hadcm3,miroc_hi,pcm]
data=[cgcm3_t47,cgcm3_t63,cnrm,echo]
elif scenario == 'a2':
#data=[ccsm,cgcm3_t47,cgcm3_t63,cnrm,csiro,echam5,echo,gfdl_2_1,hadcm3,miroc_med,mri_cgcm2,pcm]
data = [cgcm3_t47,cgcm3_t63,cnrm,echo,gfdl_2_1,miroc_med,mri_cgcm2]
for dataset in data:
file_loc = dir+'\\'+scenario+'\\'+dataset[0]
if not os.path.exists(file_loc):
os.mkdir(file_loc)
os.chdir(file_loc)
print "The current dataset being worked on is: " + dataset[0]
#The url of each dataset
dataSet = dataset[1]
#The precipitation and temperatures of each dataset
#Start at position(not index) 5 until the end of the
#dictionary's key(which is a list)
dataType = dataset[5:]
# http://cida.usgs.gov/thredds/dodsC/dcp/conus or Daily for additional aggregation/formatting (not appended yet)
timeStep = dataset[2]
#Start date
timeBegin = dataset[3]
#End date
timeEnd = dataset[4]
# data processing arguments
gmlIDs=None
verbose=True
#coverage = 'false' check if on US border/ocean
coverage = Region_return[2]
delim='COMMA'
stats='MEAN'
# run the pyGDP
start = time.time()
outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats)
end = time.time()
print 'Start time is: ' + str(start)
print 'End time is: ' + str(end)
print 'Total time was: ' + str(end-start)
print outputPath
# copy the output and rename it
shutil.copy2(outputPath, region+'_'+dataset[0]+'.csv')
ind = 5
for var in range(5, len(dataset)):
line = dataset[var].split('-')
dataset[ind] = line[2]
ind += 1
dataType = dataset[5:]
#Parse the csv file
index = 0
#parsedFiles = []
csvread = csv.reader(open(region+'_'+dataset[0] + '.csv', "rb"))
csvwrite = csv.writer(open(dataType[0] + '.csv', "wb"))
#parsedFiles.append(csvwrite)
temp = csvread
var = temp.next()
var[0] = '#'+dataType[0]
#Gets gage ids
gage = temp.next()
#Writes current variable to csv file
csvwrite.writerow(var)
#Writes all gage ids to csv file
csvwrite.writerow(gage)
for variable in dataType:
for row in csvread:
#if on last variable
if variable == dataType[len(dataType) - 1]:
csvwrite.writerow(row)
else:
if (row[0] in '#'+dataType[index+1]) or (row[0] in '# '+dataType[index+1]):
#Line 33 is used for titling the csv file the name of the variable (like tmin, ppt, or tmax)
var = '#'+dataType[index+1]
csvwrite = csv.writer(open(dataType[index+1] + '.csv', "wb"))
#parsedFiles.append(csvwrite)
row[1:] = ""
row[0] = var
csvwrite.writerow(row)
csvwrite.writerow(gage)
if len(dataType) == 2:
csvread.next()
else:
temp = csvread
csvwrite.writerow(csvread.next())
csvwrite.writerow(temp.next())
break
else:
if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]:
csvwrite.writerow(row)
print "Finished parsing " + variable + ".csv"
# use index to keep track of next variable
if (index + 1) < len(dataType):
index += 1
#pyGDP.setCSV_Parsed_Files(parsedFiles)
os.chdir(dir)
def main_func(curdir, data_set, region):
def Region_lookup(region):
region_properties = []
if region == "nhru":
region_properties.append(371)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/51d35da1e4b0ca184833940c")
region_properties.append("false")
elif region == "R01":
region_properties.append(2462)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5244735ae4b05b217bada04e")
region_properties.append("false")
elif region == "R02":
region_properties.append(4827)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52696784e4b0584cbe9168ee")
region_properties.append("false")
elif region == "R03":
region_properties.append(9899)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5283bd23e4b047efbbb57922")
region_properties.append("false")
elif region == "R04":
region_properties.append(5936)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5284ff57e4b063f258e61b9d")
region_properties.append("false")
elif region == "R05":
region_properties.append(7182)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/528516bbe4b063f258e62161")
region_properties.append("false")
elif region == "R06":
region_properties.append(2303)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/51d75296e4b055e0afd5be2c")
region_properties.append("true")
elif region == "R07":
region_properties.append(8205)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52851cd5e4b063f258e643dd")
region_properties.append("true")
elif region == "R08":
region_properties.append(4449)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52854695e4b063f258e6513c")
region_properties.append("true")
elif region == "R10L":
region_properties.append(8603)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/520031dee4b0ad2d97189db2")
region_properties.append("true")
elif region == "R10U":
region_properties.append(10299)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5216849ee4b0b45d6ba61e2e")
region_properties.append("false")
elif region == "R11":
region_properties.append(7373)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/51d1f9ebe4b08b18a62d586b")
region_properties.append("true")
elif region == "R12":
region_properties.append(7815)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5201328ae4b009d47a4c247a")
region_properties.append("false")
elif region == "R13":
region_properties.append(1958)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/51d752b9e4b055e0afd5be36")
region_properties.append("false")
elif region == "R14":
region_properties.append(3879)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52029c68e4b0e21cafa4b40c")
region_properties.append("false")
elif region == "R15":
region_properties.append(3441)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/5285389ae4b063f258e64863")
region_properties.append("false")
elif region == "R16":
region_properties.append(2664)
region_properties.append("https://www.sciencebase.gov/catalogMaps/mapping/ows/52853f97e4b063f258e64875")
region_properties.append("false")
return region_properties
def list_define(data_set):
if data_set == "PRISM":
return [
"PRISM",
"http://cida.usgs.gov/thredds/dodsC/prism",
"Monthly",
"2010-01-01T00:00:00.000Z",
"2012-12-31T00:00:00.000Z",
"ppt",
"tmx",
"tmn",
]
elif data_set == "MPI_ECHAM5":
return [
"MPI ECHAM5",
"http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/EH5/merged/Monthly/RegCM3_Monthly_merged_EH5.ncml",
"Monthly",
"1968-01-01T00:00:00.000Z",
"2099-12-31T00:00:00.000Z",
"RT",
"TA",
]
elif data_set == "GENMON":
return [
"GENMON",
"http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/GENMOM/merged/Monthly/RegCM3_Monthly_merged_GENMOM.ncml",
"Monthly",
"1980-01-01T00:00:00.000Z",
"2089-12-31T00:00:00.000Z",
"RT",
"TA",
]
elif data_set == "GFDL_CM_2_0":
return [
"GFDL CM 2.0",
"http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/GFDL/merged/Monthly/RegCM3_Monthly_merged_GFDL.ncml",
"Monthly",
"1970-01-01T00:00:00.000Z",
"2069-12-31T00:00:00.000Z",
"RT",
"TA",
]
elif data_set == "NOAA_NCEP":
return [
"NOAA NCEP",
"http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/NCEP/merged/Monthly/RegCM3_Monthly_merged_NCEP.ncml",
"Monthly",
"1982-01-01T00:00:00.000Z",
"2007-12-31T00:00:00.000Z",
"RT",
"TA",
]
elif data_set == "GSD":
return [
"Gridded Observed Data(1949-2010)",
"http://cida.usgs.gov/thredds/dodsC/new_gmo",
"Daily",
"1949-01-01T00:00:00.000Z",
"2010-12-31T00:00:00.000Z",
"pr",
"tas",
]
elif data_set == "DAYMET":
return [
"DAYMET",
"dods://cida-eros-mows1.er.usgs.gov:8080/thredds/dodsC/daymet",
"Daily",
"2010-01-01T00:00:00.000Z",
"2012-01-01T00:00:00.000Z",
"prcp",
"tmax",
"tmin",
]
elif data_set == "Idaho":
return [
"Idaho",
"http://cida.usgs.gov/thredds/dodsC/UofIMETDATA",
"Daily",
"1979-01-01T00:00:00.000Z",
"2013-01-01T00:00:00.000Z",
"precipitation_amount",
"min_air_temperature",
"max_air_temperature",
]
global csvfile, vt_dataset, nhru, length, vt_datatype, url
import pyGDP
Region_return = Region_lookup(region)
hrus = Region_return[0]
nhru = hrus
ScienceBase_URL = Region_return[1]
# NHDplus region
# link to region 6 hrus on ScienceBase
pyGDP.WFS_URL = ScienceBase_URL
url = pyGDP.WFS_URL
pyGDP = pyGDP.pyGDPwebProcessing()
# change working directory so the GDP output will be written there
# Datasets and their properties
# **********************************
# Gridded_Observed_Data_1950_1999 = ['Gridded Observed Data(1950-1999)', 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml', 'Daily', '1950-01-01T00:00:00.000Z', '1999-12-31T00:00:00.000Z', 'Prcp', 'Tavg']
# data = [PRISM]#,MPI_ECHAM5,GENMON,GFDL_CM_2_0,NOAA_NCEP,GSD,DAYMET]
# prism starts at 1895
# get list of shapefiles uploaded to the GDP
shapefiles = pyGDP.getShapefiles()
for shp in shapefiles:
print shp
# feature loaded from sciencebase
# shapefile = 'sb:'+region+'_hru'
shapefile = "sb:nhru"
user_attribute = "hru_id_loc"
user_value = None
# os.chdir('C:\\Users\\reimandy\\workspace\\userpackages\\WaterBalanceModel\\Step1_CLIMATE_DATA\\'+region+'\\SBDDS')
os.chdir(curdir)
dir = os.getcwd()
vt_data = list_define(data_set)
vt_datatype = vt_data[5:]
timestart = time.time()
file_loc = curdir + "\\" + data_set
if not os.path.exists(file_loc):
os.mkdir(file_loc)
os.chdir(file_loc)
print "The current dataset being worked on is: " + data_set
# The url of each dataset
dataSet = vt_data[1]
# The precipitation and temperatures of each dataset
# Start at position(not index) 5 until the end of the
# dictionary's key(which is a list)
dataType = vt_data[5:]
# daily or monthly for additional aggregation/formatting (not appended yet)
timeStep = vt_data[2]
# Length is for connecting to vistrails
length = timeStep
# Start date
timeBegin = vt_data[3]
# End date
timeEnd = vt_data[4]
# data processing arguments
gmlIDs = None
verbose = True
# coverage = 'false' check if on US border/ocean
coverage = Region_return[2]
delim = "COMMA"
stats = "MEAN"
# run the pyGDP
start = time.time()
outputPath = pyGDP.submitFeatureWeightedGridStatistics(
shapefile,
dataSet,
dataType,
timeBegin,
timeEnd,
user_attribute,
user_value,
gmlIDs,
verbose,
coverage,
delim,
stats,
)
end = time.time()
print "Start time is: " + str(start)
print "End time is: " + str(end)
print "Total time was: " + str(end - start)
print outputPath
# copy the output and rename it
shutil.copy2(outputPath, region + "_" + vt_data[0] + ".csv")
# #Parse the csv file
# index = 0
#
# csvread = csv.reader(open(region+'_'+dataset[0] + '.csv', "rb"))
# csvwrite = csv.writer(open(dataType[0] + '.csv', "wb"))
#
# index = 0
#
# temp = csvread
# var = temp.next()
# #Gets gage ids
# gage = temp.next()
#
# #Writes current variable to csv file
# csvwrite.writerow(var)
# #Writes all gage ids to csv file
# csvwrite.writerow(gage)
#
# for variable in dataType:
#
# for row in csvread:
# #if on last variable
# if variable == dataType[len(dataType) - 1]:
# csvwrite.writerow(row)
# else:
# if (row[0] in '#'+dataType[index+1]) or (row[0] in '# '+dataType[index+1]):
# #Line 33 is used for titling the csv file the name of the variable (like tmin, ppt, or tmax)
# var = '#'+dataType[index+1]
# csvwrite = csv.writer(open(dataType[index+1] + '.csv', "wb"))
# row[1:] = ""
# row[0] = var
# csvwrite.writerow(row)
# csvwrite.writerow(gage)
#
# if len(dataType) == 2:
# csvread.next()
# else:
# csvread.next()
# csvwrite.writerow(csvread.next())
# csvwrite.writerow(csvread.next())
# break
# else:
# if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]:
# csvwrite.writerow(row)
# print "Finished parsing " + variable + ".csv"
# parsedFiles.append(os.getcwd()+'\\'+variable+'.csv')
# # use index to keep track of next variable
# if (index + 1) < len(dataType):
# index += 1
timeend = time.time()
print "Start time of pyGDP: " + str(timestart)
print "End time of pyGDP: " + str(timeend)
print "Total time of pyGDP: " + str(timeend - timestart)
os.chdir(dir)
def main_func(region):
#NHDplus region
region = 'R06'
#region = pyGDP.getRegion()
# change working directory so the GDP output will be written there
# Datasets and their properties
#**********************************
Gridded_Observed_Data_1950_1999 = ['Gridded Observed Data(1950-1999)', 'dods://cida.usgs.gov/thredds/dodsC/gmo/GMO_w_meta.ncml', 'Daily', '1950-01-01T00:00:00.000Z', '1999-12-31T00:00:00.000Z', 'Prcp', 'Tavg']
Gridded_Observed_Data_1949_2010 = ['Gridded Observed Data(1949-2010)', 'http://cida.usgs.gov/thredds/dodsC/new_gmo', 'Daily', '1950-01-01T00:00:00.000Z', '1950-02-01T00:00:00.000Z', 'pr', 'tas']
PRISM = ['PRISM', 'http://cida.usgs.gov/thredds/dodsC/prism', 'Monthly', '1895-01-01T00:00:00.000Z', '2012-12-31T00:00:00.000Z', 'ppt', 'tmx', 'tmn']
DAYMET = ['DAYMET', 'dods://cida-eros-mows1.er.usgs.gov:8080/thredds/dodsC/daymet', 'Daily', '1980-01-01T00:00:00.000Z', '2012-01-01T00:00:00.000Z', 'prcp', 'tmax', 'tmin']
MPI_ECHAM5 = ['MPI ECHAM5', 'http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/EH5/merged/Monthly/RegCM3_Monthly_merged_EH5.ncml', 'Monthly', '1968-01-01T00:00:00.000Z', '2099-12-31T00:00:00.000Z', 'RT', 'TA']
GENMON = ['GENMON', 'http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/GENMOM/merged/Monthly/RegCM3_Monthly_merged_GENMOM.ncml', 'Monthly', '1980-01-01T00:00:00.000Z', '2089-12-31T00:00:00.000Z', 'RT', 'TA']
GFDL_CM_2_0 = ['GFDL CM 2.0', 'http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/GFDL/merged/Monthly/RegCM3_Monthly_merged_GFDL.ncml', 'Monthly', '1980-01-01T00:00:00.000Z', '2069-12-31T00:00:00.000Z', 'RT', 'TA']
NOAA_NCEP = ['NOAA NCEP', 'http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/NCEP/merged/Monthly/RegCM3_Monthly_merged_NCEP.ncml','Monthly', '1982-01-01T00:00:00.000Z', '2007-12-31T00:00:00.000Z','RT','TA']
#List dataset names
data = [Gridded_Observed_Data_1950_1999, Gridded_Observed_Data_1949_2010,PRISM,DAYMET,MPI_ECHAM5,GENMON,GFDL_CM_2_0,NOAA_NCEP]
#data=[Gridded_Observed_Data_1950_1999]
# link to region 6 hrus on ScienceBase
# put a function here to retrieve sciencebase url and number of hrus for specific region
pyGDP.WFS_URL = 'https://www.sciencebase.gov/catalogMaps/mapping/ows/51b0d374e4b030b519830d73'
#Get pyGDP.WFS_URL from vistrials
#pyGDP.WFS_URL = pyGDP.get_pyGDP_WFS_URL()
# region 6 = 2303 hrus
# for more info read:
# science base access: https://my.usgs.gov/confluence/display/GeoDataPortal/pyGDP+FAQ
# call web processing module
# will automatically use shapefiles from GDP as default unless
# you define web service
pyGDP = pyGDP.pyGDPwebProcessing()
# get list of shapefiles uploaded to the GDP
shapefiles = pyGDP.getShapefiles()
for shp in shapefiles:
print shp
# feature loaded from sciencebase
# automate region number
shapefile = 'sb:R06a_hru'
# shapefile/feature attribute for which you are summarizing info
user_attribute = 'hru_id_loc'
# create list of hru_ids that can be used for the uservalue
# need to automate number of hurs in shapefile, maybe include from scienecbase
# url retrieval above lin 25-29
# single feature id test case
#user_value = '10'
# to summarize for all ids
user_value = None
for dataset in data:
print "The current dataset being worked on is: " + dataset[0]
#The url of each dataset
dataSet = dataset[1]
#The precipitation and temperatures of each dataset
#Start at position(not index) 5 until the end of the
#dictionary's key(which is a list)
dataType=dataset[5:]
# daily or monthly for additional aggregation/formatting (not appended yet)
timeStep = dataset[2]
#Start date
timeBegin = dataset[3]
#End date
timeEnd = dataset[4]
# data processing arguments
gmlIDs=None
verbose=True
#coverage = 'false' check if on US border/ocean
coverage='true'
delim='COMMA'
stats='MEAN'
# run the pyGDP
outputPath = pyGDP.submitFeatureWeightedGridStatistics(shapefile, dataSet, dataType, timeBegin, timeEnd, user_attribute, user_value, gmlIDs, verbose, coverage, delim, stats)
print outputPath
#pyGDP.setCSV_file(outputPath)
# copy the output and rename it
shutil.copy2(outputPath, region+'_'+dataset[0]+'.csv')
csvfile = os.getcwd()+region+'_'+dataset[0]+'.csv'
#Parse the csv file
index = 0
#parsedFiles = []
csvread = csv.reader(open(outputPath + '.csv', "rb"))
csvwrite = csv.writer(open(dataType[0] + '.csv', "wb"))
temp = csvread
var = temp.next()
#Gets gage ids
gage = temp.next()
#Writes current variable to csv file
csvwrite.writerow(var)
#Writes all gage ids to csv file
csvwrite.writerow(gage)
for variable in dataType:
for row in csvread:
#if on last variable
if variable == dataType[len(dataType) - 1]:
csvwrite.writerow(row)
else:
if (row[0] in '#'+dataType[index+1]) or (row[0] in '# '+dataType[index+1]):
#Line 33 is used for titling the csv file the name of the variable (like tmin, ppt, or tmax)
var = '#'+dataType[index+1]
parsedFiles.append(os.getcwd()+'\\'+variable+'.csv')
csvwrite = csv.writer(open(dataType[index+1] + '.csv', "wb"))
row[1:] = ""
row[0] = var
csvwrite.writerow(row)
csvwrite.writerow(gage)
if len(dataType) == 2:
csvread.next()
else:
temp = csvread
csvwrite.writerow(csvread.next())
csvwrite.writerow(temp.next())
break
else:
if dataType[index+1] not in row[0] and row[0] not in dataType[index+1]:
csvwrite.writerow(row)
print "Finished parsing " + variable + ".csv"
parsedFiles.append(os.getcwd()+'\\'+variable+'.csv')
# use index to keep track of next variable
if (index + 1) < len(dataType):
index += 1