def save_ncfiles(set_num, test, ref, diff, parameter):
"""Saves the test, reference, and difference nc files."""
# Save files being plotted
# Set cdms preferences - no compression, no shuffling, no complaining
cdms2.setNetcdfDeflateFlag(1)
# 1-9, min to max - Comes at heavy IO (read/write time cost)
cdms2.setNetcdfDeflateLevelFlag(0)
cdms2.setNetcdfShuffleFlag(0)
cdms2.setCompressionWarnings(0) # Turn off warning messages
# Save test file
pth = get_output_dir(set_num, parameter)
file_test = cdms2.open(pth + '/' + parameter.output_file + '_test.nc',
'w+')
test.id = parameter.var_id
file_test.write(test)
file_test.close()
# Save reference file
file_ref = cdms2.open(pth + '/' + parameter.output_file + '_ref.nc', 'w+')
ref.id = parameter.var_id
file_ref.write(ref)
file_ref.close()
# Save difference file
file_diff = cdms2.open(pth + '/' + parameter.output_file + '_diff.nc',
'w+')
diff.id = parameter.var_id + '(test - reference)'
file_diff.write(diff)
file_diff.close()
def save_ncfiles(set_num, test, ref, diff, parameter):
"""
Saves the test, reference, and difference
data being plotted as nc files.
"""
if parameter.save_netcdf:
# Save files being plotted
# Set cdms preferences - no compression, no shuffling, no complaining
cdms2.setNetcdfDeflateFlag(1)
# 1-9, min to max - Comes at heavy IO (read/write time cost)
cdms2.setNetcdfDeflateLevelFlag(0)
cdms2.setNetcdfShuffleFlag(0)
cdms2.setCompressionWarnings(0) # Turn off warning messages
pth = get_output_dir(set_num, parameter)
# Save test file
test.id = parameter.var_id
test_pth = os.path.join(pth, parameter.output_file + '_test.nc')
with cdms2.open(test_pth, 'w+') as file_test:
file_test.write(test)
# Save reference file
ref.id = parameter.var_id
ref_pth = os.path.join(pth, parameter.output_file + '_ref.nc')
with cdms2.open(ref_pth, 'w+') as file_ref:
file_ref.write(ref)
# Save difference file
diff.id = parameter.var_id + '(test - reference)'
diff_pth = os.path.join(pth, parameter.output_file + '_diff.nc')
with cdms2.open(diff_pth, 'w+') as file_diff:
file_diff.write(diff)
def save_transient_variables_to_netcdf(set_num, variables_dict, label, parameter):
"""
Save the transient variables to nc file.
"""
if parameter.save_netcdf:
for (variable_name, variable) in variables_dict.items():
# Set cdms preferences - no compression, no shuffling, no complaining
cdms2.setNetcdfDeflateFlag(1)
# 1-9, min to max - Comes at heavy IO (read/write time cost)
cdms2.setNetcdfDeflateLevelFlag(0)
cdms2.setNetcdfShuffleFlag(0)
cdms2.setCompressionWarnings(0) # Turn off warning messages
path = get_output_dir(set_num, parameter)
# Save variable
try:
variable.id = parameter.var_id
except AttributeError:
print("Could not save variable.id for {}".format(variable_name))
file_name = "{}_{}_{}.nc".format(
parameter.output_file, variable_name, label
)
test_pth = os.path.join(path, file_name)
with cdms2.open(test_pth, "w+") as file_test:
try:
file_test.write(variable)
except AttributeError:
print("Could not write variable {}".format(variable_name))
def save_ncfiles(set_num, test, ref, diff, parameter):
"""
Saves the test, reference, and difference
data being plotted as nc files.
"""
if parameter.save_netcdf:
# Save files being plotted
# Set cdms preferences - no compression, no shuffling, no complaining
cdms2.setNetcdfDeflateFlag(1)
# 1-9, min to max - Comes at heavy IO (read/write time cost)
cdms2.setNetcdfDeflateLevelFlag(0)
cdms2.setNetcdfShuffleFlag(0)
cdms2.setCompressionWarnings(0) # Turn off warning messages
pth = get_output_dir(set_num, parameter)
# Save test file
if test.id.startswith("variable_"):
test.id = parameter.var_id
test_pth = os.path.join(pth, parameter.output_file + "_test.nc")
cdms_arg = "w"
if Path(test_pth).is_file():
cdms_arg = "a"
with cdms2.open(test_pth, cdms_arg) as file_test:
file_test.write(test)
# Save reference file
if ref.id.startswith("variable_"):
ref.id = parameter.var_id
ref_pth = os.path.join(pth, parameter.output_file + "_ref.nc")
with cdms2.open(ref_pth, cdms_arg) as file_ref:
file_ref.write(ref)
# Save difference file
if diff is not None:
if diff.id.startswith("variable_"):
diff.id = parameter.var_id + "_diff"
diff_pth = os.path.join(pth, parameter.output_file + "_diff.nc")
with cdms2.open(diff_pth, cdms_arg) as file_diff:
file_diff.write(diff)
import os
import shutil
import subprocess
import sys
import cdms2 as cdm
# Add durolib to path
sys.path.insert(1, '/export/durack1/git/pylib') # Assumes crunchy/oceanonly
from durolib import globalAttWrite
# Set cdms preferences - no compression, no shuffling, no complaining
cdm.setNetcdfDeflateFlag(1)
# 1-9, min to max - Comes at heavy IO (read/write time cost)
cdm.setNetcdfDeflateLevelFlag(9)
cdm.setNetcdfShuffleFlag(0)
cdm.setCompressionWarnings(0) # Turn off nag messages
# Set bounds automagically
# cdm.setAutoBounds(1) ; # Use with caution
# Set build info once
buildDate = '141126'
outPath = '/work/durack1/Shared/141126_metrics-acme'
# Create input variable lists
uvcdatInstall = ''.join(
['/export/durack1/', buildDate, '_pcmdi_metrics/PCMDI_METRICS/bin/'])
# Specify inputs:
# Realm ModelId InputFiles SourceDirectory
data = [
['atmos',
'ACME-CAM5-SE_v0pt1',
'B1850C5e1_ne30_',
# flake8:noqa
import cdms2
import cdutil
import MV2
# Data downloaded from here https://www.metoffice.gov.uk/hadobs/hadisst/data/download.html
# This script is to fix overlapping timebounds from original data
# also add ice mask
# Set nc classic as outputs
cdms2.setCompressionWarnings(0) # Suppress warnings
cdms2.setNetcdfShuffleFlag(0)
cdms2.setNetcdfDeflateFlag(1) # was 0 130717
cdms2.setNetcdfDeflateLevelFlag(9) # was 0 130717
cdms2.setAutoBounds(1) # Ensure bounds on time and depth axes are generated
filepath = "/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/HadISST/original_data/"
filename1 = "HadISST_ice.nc"
filename2 = "HadISST_sst.nc"
fin1 = cdms2.open(filepath + filename1)
fin2 = cdms2.open(filepath + filename2)
ice = fin1("sic")
sst = fin2("sst")
fout = cdms2.open(filepath + "HadISST_sst_ice_masked.nc", "w")
sst_masked = MV2.masked_where(ice > 0, sst, copy=True)
sst_masked.id = "sst"
cdutil.setTimeBoundsMonthly(sst_masked)
# reverse latitude so that latitude in ascending
sst_masked = sst_masked[:, ::-1, :]
fout.write(sst_masked)
import sys import time from socket import gethostname from string import replace from subprocess import call # for calling NCO functions import cdat_info import cdms2 as cdm import cdtime import cdutil as cdu import MV2 as MV # functions for dealing with masked values. import numpy import pytz # Set nc classic as outputs cdm.setCompressionWarnings(0) # Suppress warnings cdm.setNetcdfShuffleFlag(0) cdm.setNetcdfDeflateFlag(1) # was 0 130717 cdm.setNetcdfDeflateLevelFlag(9) # was 0 130717 cdm.setAutoBounds(1) # Ensure bounds on time and depth axes are generated #^^^^^^^^^^^^^^^ MODIFY FOLLOWING THINGS DEPENDING ON THE DATASET ^^^^^^^^^^^^^^^ #userinitials="CRT" userinitials = "CZ" #input_hostpath="/work/terai1/ACME/OAFLUX/" #location of input data #output_hostpath="/work/terai1/ACME/OAFLUX/" #location to save climatology files
# Python module imports
import os
import shutil
import subprocess
import sys
import cdms2
# Add durolib to path
sys.path.insert(1, '/glade/u/home/durack1')
from durolib import globalAttWrite
# Set cdms2s preferences - no compression, no shuffling, no complaining
cdms2.setNetcdfDeflateFlag(1)
# 1-9, min to max - Comes at heavy IO (read/write time cost)
cdms2.setNetcdfDeflateLevelFlag(9)
cdms2.setNetcdfShuffleFlag(0)
cdms2.setCompressionWarnings(0) # Turn off nag messages
# Set bounds automagically
# cdms2.setAutoBounds(1) ; # Use with caution
# Set build info once
buildDate = '141104'
# Create input variable lists
uvcdatInstall = ''.join(
['/glade/u/home/durack1/', buildDate, '_metrics/PCMDI_METRICS/bin/'])
data = [
[
'atmos', 'NCAR-CAM5_1deg', 'f.e12.FAMIPC5.ne30_g16.amip.002_',
'/glade/p/cgd/amp/people/hannay/amwg/climo/f.e12.FAMIPC5.ne30_g16.amip.002/0.9x1.25/'
],
[
PJD 11 Dec 2014 - Updated to latest build (including jsons)
PJD 31 Dec 2014 - Updated to latest build (jsons, and test cases)
PJD 7 Jan 2015 - Added sftlf creation
PJD 13 Jan 2015 - Updated to latest build
@author: durack1
"""
import os,string,subprocess,sys
import cdms2 as cdm
# Set cdms preferences - no compression, no shuffling, no complaining
cdm.setNetcdfDeflateFlag(0)
cdm.setNetcdfDeflateLevelFlag(0)
cdm.setNetcdfShuffleFlag(0)
cdm.setCompressionWarnings(0)
#cdm.axis.level_aliases.append('zt') ; # Add zt to axis list
#cdm.axis.latitude_aliases.append('yh') ; # Add yh to axis list
#cdm.axis.longitude_aliases.append('xh') ; # Add xh to axis list
# Set bound automagically
#cdm.setAutoBounds(1) ; # Use with caution
# Set build info once
buildDate = '150113'
# Create input variable lists
uvcdatInstall = ''.join(['/home/p1d/',buildDate,'_metrics/PCMDI_METRICS/bin/'])
data = [
['ocean','GFDL-ESM2G','experiment','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/ocean_z/av/monthly_100yr/'],
['ocean','GFDL-CM4','experiment','*0001-0005*','/archive/jpk/mdt/20140829/tikal_201403_awgUpdates_mom6_2014.08.29/CM4i_c96L48_am4a1r1_1860forc/gfdl.ncrc2-default-prod-openmp/pp/ocean_z/av/monthly_5yr/'],
['atmos','GFDL-ESM2G','experiment','*0001-0100*','/archive/esm2g/fre/postriga_esm_20110506/ESM2G/ESM2G_pi-control_C2/gfdl.default-prod/pp/atmos/av/monthly_100yr/'],
# Upload the following libraries to call in this script import argparse,datetime,gc,re,sys,time import cdat_info,cdtime,code,inspect,os,re,string,sys,pytz import cdms2 as cdm import MV2 as MV #functions for dealing with masked values. import cdutil as cdu import glob import os from socket import gethostname from string import replace import numpy from subprocess import call #for calling NCO functions # Set nc classic as outputs cdm.setCompressionWarnings(0) ; # Suppress warnings cdm.setNetcdfShuffleFlag(0) cdm.setNetcdfDeflateFlag(1) ; # was 0 130717 cdm.setNetcdfDeflateLevelFlag(9) ; # was 0 130717 cdm.setAutoBounds(1) ; # Ensure bounds on time and depth axes are generated #^^^^^^^^^^^^^^^ MODIFY FOLLOWING THINGS DEPENDING ON THE DATASET ^^^^^^^^^^^^^^^ #userinitials="CRT" userinitials="CZ" #input_hostpath="/work/terai1/ACME/OAFLUX/" #location of input data #output_hostpath="/work/terai1/ACME/OAFLUX/" #location to save climatology files data_name = 'GPCP' input_hostpath="/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/"+data_name+"/time_series/" #location of input data output_hostpath="/p/user_pub/e3sm/zhang40/analysis_data_e3sm_diags/"+data_name+"/climatology/" #location to save climatology files
import os,sys,datetime,time,glob,gc,argparse
from string import replace
from socket import gethostname
import cdms2 as cdm
import cdutil as cdu
from durolib import fixVarUnits,globalAttWrite,writeToLog
from numpy import mod
from numpy.core import shape
if 'e' in locals():
del(e,pi,sctypeNA,typeNA)
gc.collect()
# Set netcdf file criterion - turned on from default 0s
cdm.setCompressionWarnings(0) ; # Suppress warnings
cdm.setNetcdfShuffleFlag(0)
cdm.setNetcdfDeflateFlag(1)
cdm.setNetcdfDeflateLevelFlag(9)
# Hi compression: 1.4Gb file ; # Single salt variable
# No compression: 5.6Gb ; Standard (compression/shuffling): 1.5Gb ; Hi compression w/ shuffling: 1.5Gb
cdm.setAutoBounds(1) ; # Ensure bounds on time and depth axes are generated
start_time = time.time() ; # Set time counter
# Set conditional whether files are created or just numbers are calculated
parser = argparse.ArgumentParser()
parser.add_argument('model_suite',metavar='str',type=str,help='include \'cmip3/5\' as a command line argument')
parser.add_argument('experiment',metavar='str',type=str,help='including \'experiment\' will select one experiment to process')
parser.add_argument('realm',nargs='?',default='',metavar='str',type=str,help='including optional argument \'realm\' will subprocess variables')
args = parser.parse_args()
'''
Created on Mar 27, 2014
@author: tpmaxwel
'''
import cdutil, cdms2, os, numpy, copy, time, itertools
from mpi4py import MPI
from cdms2.selectors import Selector
from Utilities import *
from PVariable import PVariable
cdms2.setCompressionWarnings(False)
def recoverLostDim( array, dim_index, original_rank ):
shape = list( array.shape )
if original_rank > len( shape ):
shape.insert( dim_index, 1 )
return array.reshape( shape )
else: return array
def getBaseSizeAndShape( shape, time_index ):
prod = 1
base_shape = [1]*len(shape)
for index, sval in enumerate(shape):
if index <> time_index:
prod = prod * sval
base_shape[index] = sval
return prod, base_shape
class Task:
import numpy as np import os from tqdm import tqdm import datetime from shutil import copyfile import cdms2 from cdms2.cdscan import addAttrs from distributed import Client, as_completed, LocalCluster, get_worker import argparse cdms2.setCompressionWarnings(False) TARGETS = """CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.hfsifrazil.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.masscello.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.mlotst.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.so.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.sob.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.thetao.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.tob.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.tos.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.uo.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.vo.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.wo.gr#20191112 CMIP6.C4MIP.E3SM-Project.E3SM-1-1.hist-bgc.r1i1p1f1.Omon.zhalfo.gr#20191112 CMIP6.CMIP.E3SM-Project.E3SM-1-0.1pctCO2.r1i1p1f1.Omon.hfsifrazil.gr#20190827 CMIP6.CMIP.E3SM-Project.E3SM-1-0.1pctCO2.r1i1p1f1.Omon.masscello.gr#20190827 CMIP6.CMIP.E3SM-Project.E3SM-1-0.1pctCO2.r1i1p1f1.Omon.mlotst.gr#20190827 CMIP6.CMIP.E3SM-Project.E3SM-1-0.1pctCO2.r1i1p1f1.Omon.so.gr#20190827 CMIP6.CMIP.E3SM-Project.E3SM-1-0.1pctCO2.r1i1p1f1.Omon.sob.gr#20190827 CMIP6.CMIP.E3SM-Project.E3SM-1-0.1pctCO2.r1i1p1f1.Omon.thetao.gr#20190827 CMIP6.CMIP.E3SM-Project.E3SM-1-0.1pctCO2.r1i1p1f1.Omon.tob.gr#20190827
