def logWrite(logfile, time_since_start, path_name, i1, data_outfiles,
len_vars):
outfile_count = len(data_outfiles)
time_since_start_s = '%09.2f' % time_since_start
print "".join([
path_name.ljust(13), ' scan complete.. ',
format(i1, "1d").ljust(6), ' paths total; ',
str(outfile_count).ljust(6), ' output files to be written (',
format(len_vars, "1d").ljust(3), ' vars sampled)'
])
writeToLog(
logfile, "".join([
time_since_start_s, ' : ',
path_name.ljust(13), ' scan complete.. ',
format(i1, "1d").ljust(6), ' paths total; ',
format(outfile_count, "1d").ljust(6),
' output files to be written (',
format(len_vars, "1d").ljust(3), ' vars sampled)'
]))
return
xml_count = 6
else:
print '** HOST UNKNOWN, aborting.. **'
sys.exit()
# Set logfile attributes
time_now = datetime.datetime.now()
time_format = time_now.strftime("%y%m%d_%H%M%S")
logfile = os.path.join(
host_path, "".join([
time_format, '_make_cmip5_an-', experiment, '-', realm, '-', trim_host,
'.log'
]))
# Create logfile
if 'logfile' in locals():
writeToLog(logfile, "".join(['TIME: ', time_format]))
writeToLog(logfile, "".join(['HOSTNAME: ', host_name]))
# Get list of infiles (xml)
if all_experiments and all_realms:
filelist = glob.glob("".join([xml_path, '*/*/mo/*/*.xml']))
filelist.sort()
#print "all both"
elif all_experiments and not all_realms:
filelist = glob.glob("".join([xml_path, '*/', realm, '/mo/*/*.xml']))
filelist.sort()
#print "all experiments"
elif all_realms and not all_experiments:
filelist = glob.glob("".join([xml_path, experiment, '/*/mo/*/*.xml']))
filelist.sort()
#print "all realms"
#%% Generate log file
if experimentId == 'piControl':
experimentIdStartEndYrs = '-'.join([experimentId,''.join(['last',endYrStr])])
else:
experimentIdStartEndYrs = '-'.join([experimentId,str(startYr),str(endYr)])
timeNow = datetime.datetime.now();
timeFormat = timeNow.strftime("%y%m%dT%H%M%S")
# Test for function arg
if 'dateNow' in locals():
print('exists dateNow:',dateNow)
else:
dateNow = timeNow.strftime('%y%m%d')
print('dateNow:',dateNow)
logFile = os.path.join(workDir,'_'.join([timeFormat,'CMxWOA',mipEra,activityId,experimentIdStartEndYrs,variableId,'logs.txt']))
textToWrite = ' '.join(['TIME:',timeFormat])
writeToLog(logFile,textToWrite)
pypid = str(os.getpid()) ; # Returns calling python instance, so master also see os.getppid() - Parent
writeToLog(logFile,' '.join(['MASTER PID:',pypid]))
writeToLog(logFile,' '.join(['UV-CDAT:',sys.executable]))
host_name = gethostname()
print(' '.join(['HOSTNAME:',host_name]))
writeToLog(logFile,' '.join(['HOSTNAME:',host_name]))
print('----------')
writeToLog(logFile,'----------')
#%% Preallocate lists and fill
fileLists = []
#for mip in mipEra:
mip = mipEra
#for var in variableId:
var = variableId
# Specify default headers
header = "".join(['filename',delim,'calendar',delim,'time_units',delim,'startyr',delim,
'endyr',delim,'branch_time',delim,'branch_time_comp',delim,'branch_time_valid',delim,
'branch_time_gleckler',delim,'branch_time_gregory',delim,'piC_startyr',delim,'piC_endyr',delim,'parent_exp_id',delim,
'parent_exp_rip'])
''' Add tracking_id & creation_date
header = "".join(['filename',delim,'tracking_id',delim,'creation_date',delim,'calendar',delim,'time_units',delim,'startyear',delim,
'endyear',delim,'branch_time',delim,'branch_time_comp',delim,'branch_time_validated',delim,'parent_experiment_id',delim,'parent_experiment_rip',
delim,'contact'])
'''
# Create log file
time_now = datetime.datetime.now()
time_format = time_now.strftime("%y%m%d_%H%M%S")
logfile = os.path.join(host_path,"".join([time_format,'_make_cmip5_spawninfo-',trim_host,'.log'])) ; # BATCH MODE
writeToLog(logfile,"".join(['TIME: ',time_format]))
writeToLog(logfile,"".join(['CONTACT: Paul J. Durack, PCMDI, LLNL']))
writeToLog(logfile,"".join(['#cmip*: denotes issue with inconsistent branch time between single realisation variables']))
del(time_now,time_format)
gc.collect()
# Set check variables
filename_trim_1 = '' ; branch_time_1 = [] ; piControl_info = []; count = 0; yearfirst_1 = ''; yearlast_1 = ''
# Interrogate files for info and write to log file
for file1 in data_paths: #[0:5200]: # 130103 9877 first rcp26 file ; 130114 9748 first rcp26 file; 130318 9749 first rcp26 file; 130318 5200 include inmcm4
if mod(count,1) == 0:
print "".join(['count: ',str(count)])
count = count + 1
time_format = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
pypid = str(os.getpid()) ; # Returns calling python instance, so master also see os.getppid() - Parent
logfile = os.path.join(log_path,"".join([time_format,'_make_cmip5_xml-',trim_host,'-threads',str(threadCount),'-PID',pypid,'.log']))
if ~os.path.exists(log_path):
mkDirNoOSErr(log_path)
# Logging the explicit searched data path
os.chdir('/p/css03')
cmd = 'df -h | grep cmip5'
#sysCallTimeout(cmd,5) ; # Test for network connectivity and fail if /cmip5_css02 not alive
p = Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE)
out1,err = p.communicate()
cmd = 'df -h | grep css03'
p = Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE)
out2,err = p.communicate()
out = ''.join([out1,out2])
writeToLog(logfile,"".join(['TIME: ',time_format]))
writeToLog(logfile,"".join(['MASTER PID: ',pypid]))
print("".join(['master pid',pypid])) ; # Write master PID to sendmail/stdout
writeToLog(logfile,"".join(['UV-CDAT: ',sys.executable]))
print(''.join(['UV-CDAT: ',sys.executable])) ; # Write UV-CDAT/python path to sendmail/stdout
writeToLog(logfile,"".join(['HOSTNAME: ',host_name]))
writeToLog(logfile,"".join(['FUNCTION: ','scandir.walk']))
writeToLog(logfile,"".join(['SOURCEFILES:\n',out]))
del(trim_host,time_format,cmd,p,out,err)
gc.collect()
# Generate queue objects
manager0 = Manager()
## Mine for paths and files
queue1 = manager0.Queue(maxsize=0)
p1 = Process(target=pathToFile,args=('/p/css03/cmip5_css01/data/cmip5/output1/',start_time,queue1))
sys.exit()
# Change directory to host
os.chdir(host_path)
# Set logfile attributes
time_format = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
pypid = str(os.getpid()) ; # Returns calling python instance, so master also see os.getppid() - Parent
logfile = os.path.join(log_path,"".join([time_format,'_report_cmip5_xml-',trim_host,'-threads',str(threadCount),'-PID',pypid,'.log']))
# Logging the explicit searched data path
os.chdir('/cmip5_css02')
cmd = 'df -h | grep cmip5'
#sysCallTimeout(cmd,5) ; # Test for network connectivity and fail if /cmip5_css02 not alive
p = Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE)
out,err = p.communicate()
writeToLog(logfile,"".join(['TIME: ',time_format]))
writeToLog(logfile,"".join(['MASTER PID: ',pypid]))
print "".join(['master pid',pypid]) ; # Write master PID to sendmail/stdout
writeToLog(logfile,"".join(['HOSTNAME: ',host_name]))
writeToLog(logfile,"".join(['FUNCTION: ','scandir.walk']))
writeToLog(logfile,"".join(['SOURCEFILES:\n',out]))
del(trim_host,time_format,cmd,p,out,err)
gc.collect()
# Generate queue objects
manager0 = Manager()
## GDO2 data sources - Mine for paths and files
# gdo2_data
queue1 = manager0.Queue(maxsize=0)
p1 = Process(target=pathToFile,args=('/cmip5_gdo2/data/cmip5/',start_time,queue1))
p1.start() ; print "".join(['p1 pid: ',str(p1.ident)])
xml_count = 6
elif 'cmip5' in model_suite:
host_path = '/work/durack1/Shared/cmip5/'
xml_path = '/work/cmip5/' ; # set xml paths
xml_count = 6
else:
print '** HOST UNKNOWN, aborting.. **'
sys.exit()
# Set logfile attributes
time_now = datetime.datetime.now()
time_format = time_now.strftime("%y%m%d_%H%M%S")
logfile = os.path.join(host_path,"".join([time_format,'_make_cmip5_an-',experiment,'-',realm,'-',trim_host,'.log']))
# Create logfile
if 'logfile' in locals():
writeToLog(logfile,"".join(['TIME: ',time_format]))
writeToLog(logfile,"".join(['HOSTNAME: ',host_name]))
# Get list of infiles (xml)
if all_experiments and all_realms:
filelist = glob.glob("".join([xml_path,'*/*/mo/*/*.xml'])) ; filelist.sort()
#print "all both"
elif all_experiments and not all_realms:
filelist = glob.glob("".join([xml_path,'*/',realm,'/mo/*/*.xml'])) ; filelist.sort()
#print "all experiments"
elif all_realms and not all_experiments:
filelist = glob.glob("".join([xml_path,experiment,'/*/mo/*/*.xml'])) ; filelist.sort()
#print "all realms"
else:
filelist = glob.glob("".join([xml_path,experiment,'/',realm,'/mo/*/*.xml'])) ; filelist.sort()
#print "else"
host_path = '/work/durack1/Shared/cmip3/'
# crunchy 121129
cdat_path = '/usr/local/uvcdat/latest/bin/'
else:
print '** HOST UNKNOWN, aborting.. **'
sys.exit()
# Set logfile attributes
time_now = datetime.datetime.now()
time_format = time_now.strftime("%y%m%d_%H%M%S")
logfile = os.path.join(
host_path, "".join([
time_format, '_make_cmip3_trendsAndClims-', experiment, '-', trim_host,
'.log'
]))
writeToLog(logfile, "".join(['TIME: ', time_format, '\n']))
writeToLog(logfile, "".join(['HOSTNAME: ', host_name, '\n']))
# Get list of infiles (*.nc) and 3D (*.xml)
# Get list of infiles (xml)
if all_files:
filelist1 = glob.glob("".join([host_path, '*/*/an/*/*.nc']))
filelist2 = glob.glob("".join([host_path, '*/*/an/*/*.xml']))
else:
filelist1 = glob.glob("".join([host_path, experiment, '/*/an/*/*.nc']))
filelist2 = glob.glob("".join([host_path, experiment, '/*/an/*/*.xml']))
filelist = list(filelist1)
filelist.extend(filelist2)
filelist.sort()
del (filelist1, filelist2)
def xmlLog(logFile,fileZero,fileWarning,fileNoWrite,fileNoRead,fileNone,errorCode,inpath,outfileName,time_since_start,i,xmlBad1,xmlBad2,xmlBad3,xmlBad4,xmlBad5,xmlGood):
time_since_start_s = '%09.2f' % time_since_start
logtime_now = datetime.datetime.now()
logtime_format = logtime_now.strftime("%y%m%d_%H%M%S")
if fileZero:
# Case cdscan writes no file
if '/data/cmip5/' in inpath:
err_text = ' DATA PROBLEM 1 (cdscan error - zero infile size) indexing '
else:
err_text = ' PROBLEM 1 (cdscan error - zero infile size) indexing '
writeToLog(logFile,"".join(['** ',format(xmlBad1,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **']))
if batch_print:
print "".join(['**',err_text,inpath,' **'])
xmlBad1 = xmlBad1 + 1;
# Rename problem files
keepFile(outfileName,1)
elif fileWarning:
# Case cdscan reports an error
if '/data/cmip5/' in inpath:
err_text = "".join([' DATA PROBLEM 2 (cdscan error- \'',errorCode,'\') indexing '])
else:
err_text = "".join([' PROBLEM 2 (cdscan error - \'',errorCode,'\') indexing '])
writeToLog(logFile,"".join(['** ',format(xmlBad2,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **']))
if batch_print:
print "".join(['**',err_text,inpath,' **'])
xmlBad2 = xmlBad2 + 1;
# Rename problem files
keepFile(outfileName,2)
elif fileNoRead:
# Case cdscan reports no error, however file wasn't readable
if '/data/cmip5/' in inpath:
err_text = ' DATA PROBLEM 3 (read perms) indexing '
else:
err_text = ' PROBLEM 3 (read perms) indexing '
writeToLog(logFile,"".join(['** ',format(xmlBad3,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **']))
if batch_print:
print "".join(['**',err_text,inpath,' **'])
xmlBad3 = xmlBad3 + 1;
# Rename problem files
keepFile(outfileName,3)
elif fileNoWrite:
# Case cdscan reports no error, however file wasn't written
if '/data/cmip5/' in inpath:
err_text = ' DATA PROBLEM 4 (no outfile) indexing '
else:
err_text = ' PROBLEM 4 (no outfile) indexing '
writeToLog(logFile,"".join(['** ',format(xmlBad4,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **']))
if batch_print:
print "".join(['**',err_text,inpath,' **'])
xmlBad4 = xmlBad4 + 1;
# Rename problem files
keepFile(outfileName,4)
elif fileNone:
# Case cdscan reports no error, however file wasn't written
if '/data/cmip5/' in inpath:
err_text = ' DATA PROBLEM 5 (no infiles) indexing '
else:
err_text = ' PROBLEM 5 (no infiles) indexing '
writeToLog(logFile,"".join(['** ',format(xmlBad5,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **']))
if batch_print:
print "".join(['**',err_text,inpath,' **'])
xmlBad5 = xmlBad5 + 1;
# Rename problem files
keepFile(outfileName,5)
else:
writeToLog(logFile,"".join(['** ',format(xmlGood,"07d"),' ',logtime_format,' ',time_since_start_s,'s success creating: ',outfileName,' **']))
xmlGood = xmlGood + 1;
return[xmlBad1,xmlBad2,xmlBad3,xmlBad4,xmlBad5,xmlGood] # ; Non-parallel version of code
sys.exit()
# Change directory to host
os.chdir(host_path)
# Set logfile attributes
time_format = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
pypid = str(os.getpid()) ; # Returns calling python instance, so master also see os.getppid() - Parent
logfile = os.path.join(log_path,"".join([time_format,'_make_cmip5_xml_daily-',trim_host,'-threads',str(threadCount),'-PID',pypid,'.log']))
# Logging the explicit searched data path
os.chdir('/cmip5_css02')
cmd = 'df -h | grep cmip5'
#sysCallTimeout(cmd,5) ; # Test for network connectivity and fail if /cmip5_css02 not alive
p = Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE)
out,err = p.communicate()
writeToLog(logfile,"".join(['TIME: ',time_format]))
writeToLog(logfile,"".join(['HOSTNAME: ',host_name]))
writeToLog(logfile,"".join(['FUNCTION: ','scandir.walk']))
writeToLog(logfile,"".join(['SOURCEFILES:\n',out]))
del(trim_host,time_format,cmd,p,out,err)
gc.collect()
# Generate queue objects
manager0 = Manager()
## GDO2 data sources - Mine for paths and files
# gdo2_data
queue1 = manager0.Queue(maxsize=0)
p1 = Process(target=pathToFile,args=('/cmip5_gdo2/data/cmip5/',start_time,queue1))
p1.start() ; print "".join(['p1 pid: ',str(p1.ident)])
# gdo2_scratch
queue2 = manager0.Queue(maxsize=0)
# Manage existing dir
if os.path.exists(targetDirComp):
rmtree(targetDirComp)
os.makedirs(targetDirComp)
os.chdir(targetDirComp)
print('os.getcwd():', os.getcwd())
# Create log
timeNow = datetime.datetime.now();
timeFormat = timeNow.strftime("%y%m%dT%H%M%S")
dateNow = timeNow.strftime('%y%m%d')
#dateNow = '210226'
logFile = os.path.join(targetDirComp,'_'.join([timeFormat,'AR6WG1-Ch3-CMIP6-zostoga_log.txt']))
textToWrite = ' '.join(['TIME:',timeFormat])
writeToLog(logFile,textToWrite)
# Loop through directories and copy paths and data
for count, filepath in enumerate(filePathsKeep):
print(count, filepath)
filePathTrim = filepath.split('/')[4:]
filePathTrim = os.path.join(*filePathTrim)
print('filePathTrim: ',filePathTrim)
writeToLog(logFile,' '.join(["{:03d}".format(count), filePathTrim]))
dest = filepath.split('/')
dest = dest[4:]
dest = os.path.join(*dest)
copytree(filepath, dest)
print('copytree(', filepath, dest,')')
#%% Append log file
variableId = ['so','sos','thetao','tos']
fileLists = []
#%% Set current dir
workDir = '/work/durack1/Shared/190311_AR6/Chap3'
#%% Process variables
if cmip6:
timeNow = datetime.datetime.now();
timeFormat = timeNow.strftime("%y%m%dT%H%M%S")
logFile1 = os.path.join(workDir,'_'.join([timeFormat,'DataAvailableLog.txt']))
for expCount,expInfo in enumerate(activityExperimentId):
activityId = activityExperimentId[expInfo][0]
experimentId = activityExperimentId[expInfo][1]
print(activityId,':',experimentId)
writeToLog(logFile1,' '.join([activityId,':',experimentId]))
for var in variableId:
timeNow = datetime.datetime.now();
timeFormat = timeNow.strftime("%y%m%dT%H%M%S")
idString1 = ' '.join(['MIP:',mip,'Activity Id:',activityId,'Experiment:',experimentId,'Variable:',var])
idString2 = ' '.join([mip,activityId,experimentId,var])
logFile2 = os.path.join(workDir,'_'.join([timeFormat,idString2.replace(' ','_'),'DataAvailableLog.txt']))
writeToLog(logFile2,timeFormat)
writeToLog(logFile2,idString1)
searchPath = os.path.join(xmlPath,mip,activityId,experimentId,realm,frequency,var,'*.xml')
print('searchPath:',searchPath)
writeToLog(logFile2,' '.join(['searchPath:',searchPath]))
writeToLog(logFile1,' '.join(['searchPath:',searchPath]))
fileList = glob.glob(searchPath) ; fileList.sort()
for count,filePath in enumerate(fileList):
#print(count,filePath.split('/')[-1])
#modelSuite = 'cmip3'
#experiment = '20c3m'
#experiment = 'sresa2'
'''
# Create logfile
timeFormat = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
logPath = '/export/durack1/git/Density_bining/'
#logPath = '/work/guilyardi/git/Density_bining/'
logfile = os.path.join(
logPath, "".join([
timeFormat, '_drive_surface-', modelSuite, '-', experiment, '-',
gethostname().split('.')[0], '.log'
]))
# WORK MODE
writeToLog(logfile, "".join(['TIME: ', timeFormat]))
writeToLog(logfile, "".join(['HOSTNAME: ', gethostname()]))
print "".join(
['** Processing files from ', modelSuite, ' for ', experiment, ' **'])
writeToLog(
logfile, "".join(
['** Processing files from ', modelSuite, ' for ', experiment, ' **']))
#%%
# Set generic paths
soPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/sos')
thetaoPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/tos')
hfdsPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/hfds')
wfoPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/wfo')
tauuPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/tauuo')
tauvPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/tauvo')
def xmlLog(logFile, fileZero, fileWarning, fileNoWrite, fileNoRead, fileNone,
errorCode, inpath, outfileName, time_since_start, i, xmlBad1,
xmlBad2, xmlBad3, xmlBad4, xmlBad5, xmlGood):
time_since_start_s = '%09.2f' % time_since_start
logtime_now = datetime.datetime.now()
logtime_format = logtime_now.strftime("%y%m%d_%H%M%S")
if fileZero:
# Case cdscan writes no file
if '/data/cmip5/' in inpath:
err_text = ' DATA PROBLEM 1 (cdscan error - zero infile size) indexing '
else:
err_text = ' PROBLEM 1 (cdscan error - zero infile size) indexing '
writeToLog(
logFile, "".join([
'** ',
format(xmlBad1, "07d"), ' ', logtime_format, ' ',
time_since_start_s, 's', err_text, inpath, ' **'
]))
if batch_print:
print "".join(['**', err_text, inpath, ' **'])
xmlBad1 = xmlBad1 + 1
# Rename problem files
keepFile(outfileName, 1)
elif fileWarning:
# Case cdscan reports an error
if '/data/cmip5/' in inpath:
err_text = "".join([
' DATA PROBLEM 2 (cdscan error- \'', errorCode, '\') indexing '
])
else:
err_text = "".join(
[' PROBLEM 2 (cdscan error - \'', errorCode, '\') indexing '])
writeToLog(
logFile, "".join([
'** ',
format(xmlBad2, "07d"), ' ', logtime_format, ' ',
time_since_start_s, 's', err_text, inpath, ' **'
]))
if batch_print:
print "".join(['**', err_text, inpath, ' **'])
xmlBad2 = xmlBad2 + 1
# Rename problem files
keepFile(outfileName, 2)
elif fileNoRead:
# Case cdscan reports no error, however file wasn't readable
if '/data/cmip5/' in inpath:
err_text = ' DATA PROBLEM 3 (read perms) indexing '
else:
err_text = ' PROBLEM 3 (read perms) indexing '
writeToLog(
logFile, "".join([
'** ',
format(xmlBad3, "07d"), ' ', logtime_format, ' ',
time_since_start_s, 's', err_text, inpath, ' **'
]))
if batch_print:
print "".join(['**', err_text, inpath, ' **'])
xmlBad3 = xmlBad3 + 1
# Rename problem files
keepFile(outfileName, 3)
elif fileNoWrite:
# Case cdscan reports no error, however file wasn't written
if '/data/cmip5/' in inpath:
err_text = ' DATA PROBLEM 4 (no outfile) indexing '
else:
err_text = ' PROBLEM 4 (no outfile) indexing '
writeToLog(
logFile, "".join([
'** ',
format(xmlBad4, "07d"), ' ', logtime_format, ' ',
time_since_start_s, 's', err_text, inpath, ' **'
]))
if batch_print:
print "".join(['**', err_text, inpath, ' **'])
xmlBad4 = xmlBad4 + 1
# Rename problem files
keepFile(outfileName, 4)
elif fileNone:
# Case cdscan reports no error, however file wasn't written
if '/data/cmip5/' in inpath:
err_text = ' DATA PROBLEM 5 (no infiles) indexing '
else:
err_text = ' PROBLEM 5 (no infiles) indexing '
writeToLog(
logFile, "".join([
'** ',
format(xmlBad5, "07d"), ' ', logtime_format, ' ',
time_since_start_s, 's', err_text, inpath, ' **'
]))
if batch_print:
print "".join(['**', err_text, inpath, ' **'])
xmlBad5 = xmlBad5 + 1
# Rename problem files
keepFile(outfileName, 5)
else:
writeToLog(
logFile, "".join([
'** ',
format(xmlGood, "07d"), ' ', logtime_format, ' ',
time_since_start_s, 's success creating: ', outfileName, ' **'
]))
xmlGood = xmlGood + 1
return [xmlBad1, xmlBad2, xmlBad3, xmlBad4, xmlBad5,
xmlGood] # ; Non-parallel version of code
from durolib import fixVarUnits, globalAttWrite, writeToLog # ,trimModelList
# %% Set current dirs
workDur = "/p/user_pub/climate_work/durack1"
workDir = os.path.join(workDur, "Shared/210128_PaperPlots_Rothigetal/")
xmlPath = "/p/user_pub/xclim/"
# '/data_crunchy_oceanonly/crunchy_work/cmip-dyn'
# %% Generate log file
timeNow = datetime.datetime.now()
timeFormat = timeNow.strftime("%y%m%dT%H%M%S")
dateNow = timeNow.strftime("%y%m%d")
# dateNow = '210226'
logFile = os.path.join(workDir, "_".join([timeFormat, "RothigData_log.txt"]))
textToWrite = " ".join(["TIME:", timeFormat])
writeToLog(logFile, textToWrite)
pypid = str(os.getpid())
# Returns calling python instance, so master also see os.getppid() - Parent
writeToLog(logFile, " ".join(["MASTER PID:", pypid]))
writeToLog(logFile, " ".join(["UV-CDAT:", sys.executable]))
host_name = gethostname()
print(" ".join(["HOSTNAME:", host_name]))
writeToLog(logFile, " ".join(["HOSTNAME:", host_name]))
print("----------")
writeToLog(logFile, "----------")
# %% Preallocate lists and fill
fileLists = []
mipEra = "CMIP6"
actExpPair = {}
actExpPair["CMIP"] = {}
def logWrite(logfile,time_since_start,path_name,i1,data_outfiles,len_vars):
outfile_count = len(data_outfiles)
time_since_start_s = '%09.2f' % time_since_start
print "".join([path_name.ljust(13),' scan complete.. ',format(i1,"1d").ljust(6),' paths total; ',str(outfile_count).ljust(6),' output files to be written (',format(len_vars,"1d").ljust(3),' vars sampled)'])
writeToLog(logfile,"".join([time_since_start_s,' : ',path_name.ljust(13),' scan complete.. ',format(i1,"1d").ljust(6),' paths total; ',format(outfile_count,"1d").ljust(6),' output files to be written (',format(len_vars,"1d").ljust(3),' vars sampled)']))
return
# Set host information and directories
host_name = gethostname()
if host_name in {'crunchy.llnl.gov','oceanonly.llnl.gov'}:
trim_host = replace(host_name,'.llnl.gov','')
host_path = '/work/durack1/Shared/cmip3/' ; # crunchy 121129
cdat_path = '/usr/local/uvcdat/latest/bin/'
else:
print '** HOST UNKNOWN, aborting.. **'
sys.exit()
# Set logfile attributes
time_now = datetime.datetime.now()
time_format = time_now.strftime("%y%m%d_%H%M%S")
logfile = os.path.join(host_path,"".join([time_format,'_make_cmip3_trendsAndClims-',experiment,'-',trim_host,'.log']))
writeToLog(logfile,"".join(['TIME: ',time_format,'\n']))
writeToLog(logfile,"".join(['HOSTNAME: ',host_name,'\n']))
# Get list of infiles (*.nc) and 3D (*.xml)
# Get list of infiles (xml)
if all_files:
filelist1 = glob.glob("".join([host_path,'*/*/an/*/*.nc']))
filelist2 = glob.glob("".join([host_path,'*/*/an/*/*.xml']))
else:
filelist1 = glob.glob("".join([host_path,experiment,'/*/an/*/*.nc']))
filelist2 = glob.glob("".join([host_path,experiment,'/*/an/*/*.xml']))
filelist = list(filelist1)
filelist.extend(filelist2) ; filelist.sort()
del(filelist1,filelist2)
gc.collect()
pathin = '/work/cmip3'
elif 'cmip5' in model_suite:
host_path = '/work/durack1/Shared/cmip5/'
pathin = '/work/cmip5' ; # set xml paths
# Load drift dictionary
cmip5_branch_time_dict = pickle.load(open(os.path.join(host_path,"".join([model_suite,'_branch_time_dict.pickle'])),'rb'))
else:
print '** HOST UNKNOWN, aborting.. **'
sys.exit()
# Set logfile attributes
time_format = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
logfile = os.path.join(host_path,"".join([time_format,'_make_',model_suite,'_trendsAndClims-',experiment,'-',realm,'-',variable,'-',time_yrs,'-',trim_host,'.log']))
#logfile = os.path.join(host_path,'tmp',"".join([time_format,'_make_',model_suite,'_trendsAndClims-',experiment,'-',realm,'-',variable,'-',time_yrs,'-',trim_host,'.log'])) ; ## TEST ##
# Create logfile
writeToLog(logfile,"".join(['TIME: ',time_format]))
writeToLog(logfile,"".join(['HOSTNAME: ',host_name]))
del(time_format) ; gc.collect()
#%% Get list of infiles (*.nc) and 3D (*.xml)
if all_files and all_realms:
print 'all_files and all_realms'
filelist1 = glob.glob(os.path.join(host_path,'*/*/an/*/*.nc'))
filelist2 = glob.glob(os.path.join(host_path,'*/*/an/*/*.xml'))
elif all_realms:
print 'all_realms'
filelist1 = glob.glob(os.path.join(host_path,experiment,'*/an/*/*.nc'))
filelist2 = glob.glob(os.path.join(host_path,experiment,'*/an/*/*.xml'))
elif variable not in 'all':
print ''.join(['variable: ',variable])
filelist1 = glob.glob(os.path.join(host_path,experiment,'*/an',variable,'*.nc'))
#modelSuite = 'cmip5'
#experiment = 'historical'
#experiment = 'rcp85'
#modelSuite = 'cmip3'
#experiment = '20c3m'
#experiment = 'sresa2'
#%%
# Create logfile
timeFormat = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
if os.getlogin() == 'durack1':
logPath = '/export/durack1/git/Density_bining/'
elif os.getlogin() == 'eguil':
logPath = '/work/guilyardi/git/Density_bining/'
logfile = os.path.join(logPath,"".join([timeFormat,'_drive_density-',modelSuite,'-',experiment,'-',gethostname().split('.')[0],'.log'])) ; # WORK MODE
writeToLog(logfile,"".join(['TIME: ',timeFormat]))
writeToLog(logfile,"".join(['HOSTNAME: ',gethostname()]))
print "".join(['** Processing files from ',modelSuite,' for ',experiment,' **'])
writeToLog(logfile,"".join(['** Processing files from ',modelSuite,' for ',experiment,' **']))
#%%
# Set generic paths
soPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/so');
thetaoPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/thetao');
fxPath = os.path.join('/work',modelSuite,'fx/fx/areacello');
# Validate paths
if not os.path.exists(outPath):
pass
#os.makedirs(outPath)
if not os.path.exists(soPath) or not os.path.exists(thetaoPath) or not os.path.exists(fxPath):
# Create logfile
time_now = datetime.datetime.now()
time_format = time_now.strftime("%y%m%d_%H%M%S")
logfile = os.path.join(
host_path, "".join([
time_format, '_make_', model_suite, '_an3D-', exp, '-', var, '-',
trim_host, '.log'
]))
# Log disk usage info for $machine:/work
os.chdir('/work')
cmd = 'df -h /work'
o = os.popen(cmd).readlines()
oj = "".join(o)
# Open logfile to write
writeToLog(logfile, "".join(['TIME: ', time_format, '\n']))
writeToLog(logfile, "".join(['HOSTNAME: ', host_name, '\n']))
writeToLog(logfile, "".join(['WORKDISK: ', oj]))
# Set inputs
indir = os.path.join(pathin, exp, 'ocn/mo', var) # *.xmls
outdir = os.path.join(host_path, exp, 'ocn/an', var, 'ncs')
# Get input xml files
lst = os.listdir(indir)
if reverse:
lst.sort(reverse=True)
else:
lst.sort()
# Loop over inputs
experiment = 'historical'
outPath = '/work/guilyardi/Shared/data_density/test'
#experiment = 'rcp85'
#outPath = '/work/guilyardi/git/Density_bining/test_cmip5'
#outPath = os.path.join('/work/durack1/Shared/data_density',datetime.datetime.now().strftime("%y%m%d"));
#modelSuite = 'cmip3'
#experiment = '20c3m'
#experiment = 'sresa2'
'''
# Create logfile
timeFormat = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
logPath = '/export/durack1/git/Density_bining/'
#logPath = '/work/guilyardi/git/Density_bining/'
logfile = os.path.join(logPath,"".join([timeFormat,'_drive_surface-',modelSuite,'-',experiment,'-',gethostname().split('.')[0],'.log'])) ; # WORK MODE
writeToLog(logfile,"".join(['TIME: ',timeFormat]))
writeToLog(logfile,"".join(['HOSTNAME: ',gethostname()]))
print "".join(['** Processing files from ',modelSuite,' for ',experiment,' **'])
writeToLog(logfile,"".join(['** Processing files from ',modelSuite,' for ',experiment,' **']))
#%%
# Set generic paths
soPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/sos');
thetaoPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/tos');
hfdsPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/hfds');
wfoPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/wfo');
tauuPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/tauuo');
tauvPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/tauvo');
fxPath = os.path.join('/work',modelSuite,'fx/fx/areacello');
# Validate paths
model_suite = 'cmip5' ; ##TEST_MODE##
var = 'uo' ; ##TEST_MODE##
exp = 'historical' ; ##TEST_MODE##
'''
# Create logfile
time_now = datetime.datetime.now()
time_format = time_now.strftime("%y%m%d_%H%M%S")
logfile = os.path.join(host_path,"".join([time_format,'_make_',model_suite,'_an3D-',exp,'-',var,'-',trim_host,'.log']))
# Log disk usage info for $machine:/work
os.chdir('/work')
cmd = 'df -h /work'
o = os.popen(cmd).readlines()
oj = "".join(o)
# Open logfile to write
writeToLog(logfile,"".join(['TIME: ',time_format,'\n']))
writeToLog(logfile,"".join(['HOSTNAME: ',host_name,'\n']))
writeToLog(logfile,"".join(['WORKDISK: ',oj]))
# Set inputs
indir = os.path.join(pathin,exp,'ocn/mo',var) # *.xmls
outdir = os.path.join(host_path,exp,'ocn/an',var,'ncs')
# Get input xml files
lst = os.listdir(indir);
if reverse:
lst.sort(reverse=True);
else:
lst.sort()
# Loop over inputs
