def logWrite(logfile, time_since_start, path_name, i1, data_outfiles, len_vars): outfile_count = len(data_outfiles) time_since_start_s = '%09.2f' % time_since_start print "".join([ path_name.ljust(13), ' scan complete.. ', format(i1, "1d").ljust(6), ' paths total; ', str(outfile_count).ljust(6), ' output files to be written (', format(len_vars, "1d").ljust(3), ' vars sampled)' ]) writeToLog( logfile, "".join([ time_since_start_s, ' : ', path_name.ljust(13), ' scan complete.. ', format(i1, "1d").ljust(6), ' paths total; ', format(outfile_count, "1d").ljust(6), ' output files to be written (', format(len_vars, "1d").ljust(3), ' vars sampled)' ])) return
xml_count = 6 else: print '** HOST UNKNOWN, aborting.. **' sys.exit() # Set logfile attributes time_now = datetime.datetime.now() time_format = time_now.strftime("%y%m%d_%H%M%S") logfile = os.path.join( host_path, "".join([ time_format, '_make_cmip5_an-', experiment, '-', realm, '-', trim_host, '.log' ])) # Create logfile if 'logfile' in locals(): writeToLog(logfile, "".join(['TIME: ', time_format])) writeToLog(logfile, "".join(['HOSTNAME: ', host_name])) # Get list of infiles (xml) if all_experiments and all_realms: filelist = glob.glob("".join([xml_path, '*/*/mo/*/*.xml'])) filelist.sort() #print "all both" elif all_experiments and not all_realms: filelist = glob.glob("".join([xml_path, '*/', realm, '/mo/*/*.xml'])) filelist.sort() #print "all experiments" elif all_realms and not all_experiments: filelist = glob.glob("".join([xml_path, experiment, '/*/mo/*/*.xml'])) filelist.sort() #print "all realms"
#%% Generate log file if experimentId == 'piControl': experimentIdStartEndYrs = '-'.join([experimentId,''.join(['last',endYrStr])]) else: experimentIdStartEndYrs = '-'.join([experimentId,str(startYr),str(endYr)]) timeNow = datetime.datetime.now(); timeFormat = timeNow.strftime("%y%m%dT%H%M%S") # Test for function arg if 'dateNow' in locals(): print('exists dateNow:',dateNow) else: dateNow = timeNow.strftime('%y%m%d') print('dateNow:',dateNow) logFile = os.path.join(workDir,'_'.join([timeFormat,'CMxWOA',mipEra,activityId,experimentIdStartEndYrs,variableId,'logs.txt'])) textToWrite = ' '.join(['TIME:',timeFormat]) writeToLog(logFile,textToWrite) pypid = str(os.getpid()) ; # Returns calling python instance, so master also see os.getppid() - Parent writeToLog(logFile,' '.join(['MASTER PID:',pypid])) writeToLog(logFile,' '.join(['UV-CDAT:',sys.executable])) host_name = gethostname() print(' '.join(['HOSTNAME:',host_name])) writeToLog(logFile,' '.join(['HOSTNAME:',host_name])) print('----------') writeToLog(logFile,'----------') #%% Preallocate lists and fill fileLists = [] #for mip in mipEra: mip = mipEra #for var in variableId: var = variableId
# Specify default headers header = "".join(['filename',delim,'calendar',delim,'time_units',delim,'startyr',delim, 'endyr',delim,'branch_time',delim,'branch_time_comp',delim,'branch_time_valid',delim, 'branch_time_gleckler',delim,'branch_time_gregory',delim,'piC_startyr',delim,'piC_endyr',delim,'parent_exp_id',delim, 'parent_exp_rip']) ''' Add tracking_id & creation_date header = "".join(['filename',delim,'tracking_id',delim,'creation_date',delim,'calendar',delim,'time_units',delim,'startyear',delim, 'endyear',delim,'branch_time',delim,'branch_time_comp',delim,'branch_time_validated',delim,'parent_experiment_id',delim,'parent_experiment_rip', delim,'contact']) ''' # Create log file time_now = datetime.datetime.now() time_format = time_now.strftime("%y%m%d_%H%M%S") logfile = os.path.join(host_path,"".join([time_format,'_make_cmip5_spawninfo-',trim_host,'.log'])) ; # BATCH MODE writeToLog(logfile,"".join(['TIME: ',time_format])) writeToLog(logfile,"".join(['CONTACT: Paul J. Durack, PCMDI, LLNL'])) writeToLog(logfile,"".join(['#cmip*: denotes issue with inconsistent branch time between single realisation variables'])) del(time_now,time_format) gc.collect() # Set check variables filename_trim_1 = '' ; branch_time_1 = [] ; piControl_info = []; count = 0; yearfirst_1 = ''; yearlast_1 = '' # Interrogate files for info and write to log file for file1 in data_paths: #[0:5200]: # 130103 9877 first rcp26 file ; 130114 9748 first rcp26 file; 130318 9749 first rcp26 file; 130318 5200 include inmcm4 if mod(count,1) == 0: print "".join(['count: ',str(count)]) count = count + 1
time_format = datetime.datetime.now().strftime("%y%m%d_%H%M%S") pypid = str(os.getpid()) ; # Returns calling python instance, so master also see os.getppid() - Parent logfile = os.path.join(log_path,"".join([time_format,'_make_cmip5_xml-',trim_host,'-threads',str(threadCount),'-PID',pypid,'.log'])) if ~os.path.exists(log_path): mkDirNoOSErr(log_path) # Logging the explicit searched data path os.chdir('/p/css03') cmd = 'df -h | grep cmip5' #sysCallTimeout(cmd,5) ; # Test for network connectivity and fail if /cmip5_css02 not alive p = Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE) out1,err = p.communicate() cmd = 'df -h | grep css03' p = Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE) out2,err = p.communicate() out = ''.join([out1,out2]) writeToLog(logfile,"".join(['TIME: ',time_format])) writeToLog(logfile,"".join(['MASTER PID: ',pypid])) print("".join(['master pid',pypid])) ; # Write master PID to sendmail/stdout writeToLog(logfile,"".join(['UV-CDAT: ',sys.executable])) print(''.join(['UV-CDAT: ',sys.executable])) ; # Write UV-CDAT/python path to sendmail/stdout writeToLog(logfile,"".join(['HOSTNAME: ',host_name])) writeToLog(logfile,"".join(['FUNCTION: ','scandir.walk'])) writeToLog(logfile,"".join(['SOURCEFILES:\n',out])) del(trim_host,time_format,cmd,p,out,err) gc.collect() # Generate queue objects manager0 = Manager() ## Mine for paths and files queue1 = manager0.Queue(maxsize=0) p1 = Process(target=pathToFile,args=('/p/css03/cmip5_css01/data/cmip5/output1/',start_time,queue1))
sys.exit() # Change directory to host os.chdir(host_path) # Set logfile attributes time_format = datetime.datetime.now().strftime("%y%m%d_%H%M%S") pypid = str(os.getpid()) ; # Returns calling python instance, so master also see os.getppid() - Parent logfile = os.path.join(log_path,"".join([time_format,'_report_cmip5_xml-',trim_host,'-threads',str(threadCount),'-PID',pypid,'.log'])) # Logging the explicit searched data path os.chdir('/cmip5_css02') cmd = 'df -h | grep cmip5' #sysCallTimeout(cmd,5) ; # Test for network connectivity and fail if /cmip5_css02 not alive p = Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE) out,err = p.communicate() writeToLog(logfile,"".join(['TIME: ',time_format])) writeToLog(logfile,"".join(['MASTER PID: ',pypid])) print "".join(['master pid',pypid]) ; # Write master PID to sendmail/stdout writeToLog(logfile,"".join(['HOSTNAME: ',host_name])) writeToLog(logfile,"".join(['FUNCTION: ','scandir.walk'])) writeToLog(logfile,"".join(['SOURCEFILES:\n',out])) del(trim_host,time_format,cmd,p,out,err) gc.collect() # Generate queue objects manager0 = Manager() ## GDO2 data sources - Mine for paths and files # gdo2_data queue1 = manager0.Queue(maxsize=0) p1 = Process(target=pathToFile,args=('/cmip5_gdo2/data/cmip5/',start_time,queue1)) p1.start() ; print "".join(['p1 pid: ',str(p1.ident)])
xml_count = 6 elif 'cmip5' in model_suite: host_path = '/work/durack1/Shared/cmip5/' xml_path = '/work/cmip5/' ; # set xml paths xml_count = 6 else: print '** HOST UNKNOWN, aborting.. **' sys.exit() # Set logfile attributes time_now = datetime.datetime.now() time_format = time_now.strftime("%y%m%d_%H%M%S") logfile = os.path.join(host_path,"".join([time_format,'_make_cmip5_an-',experiment,'-',realm,'-',trim_host,'.log'])) # Create logfile if 'logfile' in locals(): writeToLog(logfile,"".join(['TIME: ',time_format])) writeToLog(logfile,"".join(['HOSTNAME: ',host_name])) # Get list of infiles (xml) if all_experiments and all_realms: filelist = glob.glob("".join([xml_path,'*/*/mo/*/*.xml'])) ; filelist.sort() #print "all both" elif all_experiments and not all_realms: filelist = glob.glob("".join([xml_path,'*/',realm,'/mo/*/*.xml'])) ; filelist.sort() #print "all experiments" elif all_realms and not all_experiments: filelist = glob.glob("".join([xml_path,experiment,'/*/mo/*/*.xml'])) ; filelist.sort() #print "all realms" else: filelist = glob.glob("".join([xml_path,experiment,'/',realm,'/mo/*/*.xml'])) ; filelist.sort() #print "else"
host_path = '/work/durack1/Shared/cmip3/' # crunchy 121129 cdat_path = '/usr/local/uvcdat/latest/bin/' else: print '** HOST UNKNOWN, aborting.. **' sys.exit() # Set logfile attributes time_now = datetime.datetime.now() time_format = time_now.strftime("%y%m%d_%H%M%S") logfile = os.path.join( host_path, "".join([ time_format, '_make_cmip3_trendsAndClims-', experiment, '-', trim_host, '.log' ])) writeToLog(logfile, "".join(['TIME: ', time_format, '\n'])) writeToLog(logfile, "".join(['HOSTNAME: ', host_name, '\n'])) # Get list of infiles (*.nc) and 3D (*.xml) # Get list of infiles (xml) if all_files: filelist1 = glob.glob("".join([host_path, '*/*/an/*/*.nc'])) filelist2 = glob.glob("".join([host_path, '*/*/an/*/*.xml'])) else: filelist1 = glob.glob("".join([host_path, experiment, '/*/an/*/*.nc'])) filelist2 = glob.glob("".join([host_path, experiment, '/*/an/*/*.xml'])) filelist = list(filelist1) filelist.extend(filelist2) filelist.sort() del (filelist1, filelist2)
def xmlLog(logFile,fileZero,fileWarning,fileNoWrite,fileNoRead,fileNone,errorCode,inpath,outfileName,time_since_start,i,xmlBad1,xmlBad2,xmlBad3,xmlBad4,xmlBad5,xmlGood): time_since_start_s = '%09.2f' % time_since_start logtime_now = datetime.datetime.now() logtime_format = logtime_now.strftime("%y%m%d_%H%M%S") if fileZero: # Case cdscan writes no file if '/data/cmip5/' in inpath: err_text = ' DATA PROBLEM 1 (cdscan error - zero infile size) indexing ' else: err_text = ' PROBLEM 1 (cdscan error - zero infile size) indexing ' writeToLog(logFile,"".join(['** ',format(xmlBad1,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **'])) if batch_print: print "".join(['**',err_text,inpath,' **']) xmlBad1 = xmlBad1 + 1; # Rename problem files keepFile(outfileName,1) elif fileWarning: # Case cdscan reports an error if '/data/cmip5/' in inpath: err_text = "".join([' DATA PROBLEM 2 (cdscan error- \'',errorCode,'\') indexing ']) else: err_text = "".join([' PROBLEM 2 (cdscan error - \'',errorCode,'\') indexing ']) writeToLog(logFile,"".join(['** ',format(xmlBad2,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **'])) if batch_print: print "".join(['**',err_text,inpath,' **']) xmlBad2 = xmlBad2 + 1; # Rename problem files keepFile(outfileName,2) elif fileNoRead: # Case cdscan reports no error, however file wasn't readable if '/data/cmip5/' in inpath: err_text = ' DATA PROBLEM 3 (read perms) indexing ' else: err_text = ' PROBLEM 3 (read perms) indexing ' writeToLog(logFile,"".join(['** ',format(xmlBad3,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **'])) if batch_print: print "".join(['**',err_text,inpath,' **']) xmlBad3 = xmlBad3 + 1; # Rename problem files keepFile(outfileName,3) elif fileNoWrite: # Case cdscan reports no error, however file wasn't written if '/data/cmip5/' in inpath: err_text = ' DATA PROBLEM 4 (no outfile) indexing ' else: err_text = ' PROBLEM 4 (no outfile) indexing ' writeToLog(logFile,"".join(['** ',format(xmlBad4,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **'])) if batch_print: print "".join(['**',err_text,inpath,' **']) xmlBad4 = xmlBad4 + 1; # Rename problem files keepFile(outfileName,4) elif fileNone: # Case cdscan reports no error, however file wasn't written if '/data/cmip5/' in inpath: err_text = ' DATA PROBLEM 5 (no infiles) indexing ' else: err_text = ' PROBLEM 5 (no infiles) indexing ' writeToLog(logFile,"".join(['** ',format(xmlBad5,"07d"),' ',logtime_format,' ',time_since_start_s,'s',err_text,inpath,' **'])) if batch_print: print "".join(['**',err_text,inpath,' **']) xmlBad5 = xmlBad5 + 1; # Rename problem files keepFile(outfileName,5) else: writeToLog(logFile,"".join(['** ',format(xmlGood,"07d"),' ',logtime_format,' ',time_since_start_s,'s success creating: ',outfileName,' **'])) xmlGood = xmlGood + 1; return[xmlBad1,xmlBad2,xmlBad3,xmlBad4,xmlBad5,xmlGood] # ; Non-parallel version of code
sys.exit() # Change directory to host os.chdir(host_path) # Set logfile attributes time_format = datetime.datetime.now().strftime("%y%m%d_%H%M%S") pypid = str(os.getpid()) ; # Returns calling python instance, so master also see os.getppid() - Parent logfile = os.path.join(log_path,"".join([time_format,'_make_cmip5_xml_daily-',trim_host,'-threads',str(threadCount),'-PID',pypid,'.log'])) # Logging the explicit searched data path os.chdir('/cmip5_css02') cmd = 'df -h | grep cmip5' #sysCallTimeout(cmd,5) ; # Test for network connectivity and fail if /cmip5_css02 not alive p = Popen(cmd,shell=True,stdout=PIPE,stderr=PIPE) out,err = p.communicate() writeToLog(logfile,"".join(['TIME: ',time_format])) writeToLog(logfile,"".join(['HOSTNAME: ',host_name])) writeToLog(logfile,"".join(['FUNCTION: ','scandir.walk'])) writeToLog(logfile,"".join(['SOURCEFILES:\n',out])) del(trim_host,time_format,cmd,p,out,err) gc.collect() # Generate queue objects manager0 = Manager() ## GDO2 data sources - Mine for paths and files # gdo2_data queue1 = manager0.Queue(maxsize=0) p1 = Process(target=pathToFile,args=('/cmip5_gdo2/data/cmip5/',start_time,queue1)) p1.start() ; print "".join(['p1 pid: ',str(p1.ident)]) # gdo2_scratch queue2 = manager0.Queue(maxsize=0)
# Manage existing dir if os.path.exists(targetDirComp): rmtree(targetDirComp) os.makedirs(targetDirComp) os.chdir(targetDirComp) print('os.getcwd():', os.getcwd()) # Create log timeNow = datetime.datetime.now(); timeFormat = timeNow.strftime("%y%m%dT%H%M%S") dateNow = timeNow.strftime('%y%m%d') #dateNow = '210226' logFile = os.path.join(targetDirComp,'_'.join([timeFormat,'AR6WG1-Ch3-CMIP6-zostoga_log.txt'])) textToWrite = ' '.join(['TIME:',timeFormat]) writeToLog(logFile,textToWrite) # Loop through directories and copy paths and data for count, filepath in enumerate(filePathsKeep): print(count, filepath) filePathTrim = filepath.split('/')[4:] filePathTrim = os.path.join(*filePathTrim) print('filePathTrim: ',filePathTrim) writeToLog(logFile,' '.join(["{:03d}".format(count), filePathTrim])) dest = filepath.split('/') dest = dest[4:] dest = os.path.join(*dest) copytree(filepath, dest) print('copytree(', filepath, dest,')') #%% Append log file
variableId = ['so','sos','thetao','tos'] fileLists = [] #%% Set current dir workDir = '/work/durack1/Shared/190311_AR6/Chap3' #%% Process variables if cmip6: timeNow = datetime.datetime.now(); timeFormat = timeNow.strftime("%y%m%dT%H%M%S") logFile1 = os.path.join(workDir,'_'.join([timeFormat,'DataAvailableLog.txt'])) for expCount,expInfo in enumerate(activityExperimentId): activityId = activityExperimentId[expInfo][0] experimentId = activityExperimentId[expInfo][1] print(activityId,':',experimentId) writeToLog(logFile1,' '.join([activityId,':',experimentId])) for var in variableId: timeNow = datetime.datetime.now(); timeFormat = timeNow.strftime("%y%m%dT%H%M%S") idString1 = ' '.join(['MIP:',mip,'Activity Id:',activityId,'Experiment:',experimentId,'Variable:',var]) idString2 = ' '.join([mip,activityId,experimentId,var]) logFile2 = os.path.join(workDir,'_'.join([timeFormat,idString2.replace(' ','_'),'DataAvailableLog.txt'])) writeToLog(logFile2,timeFormat) writeToLog(logFile2,idString1) searchPath = os.path.join(xmlPath,mip,activityId,experimentId,realm,frequency,var,'*.xml') print('searchPath:',searchPath) writeToLog(logFile2,' '.join(['searchPath:',searchPath])) writeToLog(logFile1,' '.join(['searchPath:',searchPath])) fileList = glob.glob(searchPath) ; fileList.sort() for count,filePath in enumerate(fileList): #print(count,filePath.split('/')[-1])
#modelSuite = 'cmip3' #experiment = '20c3m' #experiment = 'sresa2' ''' # Create logfile timeFormat = datetime.datetime.now().strftime("%y%m%d_%H%M%S") logPath = '/export/durack1/git/Density_bining/' #logPath = '/work/guilyardi/git/Density_bining/' logfile = os.path.join( logPath, "".join([ timeFormat, '_drive_surface-', modelSuite, '-', experiment, '-', gethostname().split('.')[0], '.log' ])) # WORK MODE writeToLog(logfile, "".join(['TIME: ', timeFormat])) writeToLog(logfile, "".join(['HOSTNAME: ', gethostname()])) print "".join( ['** Processing files from ', modelSuite, ' for ', experiment, ' **']) writeToLog( logfile, "".join( ['** Processing files from ', modelSuite, ' for ', experiment, ' **'])) #%% # Set generic paths soPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/sos') thetaoPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/tos') hfdsPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/hfds') wfoPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/wfo') tauuPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/tauuo') tauvPath = os.path.join('/work', modelSuite, experiment, 'ocn/mo/tauvo')
def xmlLog(logFile, fileZero, fileWarning, fileNoWrite, fileNoRead, fileNone, errorCode, inpath, outfileName, time_since_start, i, xmlBad1, xmlBad2, xmlBad3, xmlBad4, xmlBad5, xmlGood): time_since_start_s = '%09.2f' % time_since_start logtime_now = datetime.datetime.now() logtime_format = logtime_now.strftime("%y%m%d_%H%M%S") if fileZero: # Case cdscan writes no file if '/data/cmip5/' in inpath: err_text = ' DATA PROBLEM 1 (cdscan error - zero infile size) indexing ' else: err_text = ' PROBLEM 1 (cdscan error - zero infile size) indexing ' writeToLog( logFile, "".join([ '** ', format(xmlBad1, "07d"), ' ', logtime_format, ' ', time_since_start_s, 's', err_text, inpath, ' **' ])) if batch_print: print "".join(['**', err_text, inpath, ' **']) xmlBad1 = xmlBad1 + 1 # Rename problem files keepFile(outfileName, 1) elif fileWarning: # Case cdscan reports an error if '/data/cmip5/' in inpath: err_text = "".join([ ' DATA PROBLEM 2 (cdscan error- \'', errorCode, '\') indexing ' ]) else: err_text = "".join( [' PROBLEM 2 (cdscan error - \'', errorCode, '\') indexing ']) writeToLog( logFile, "".join([ '** ', format(xmlBad2, "07d"), ' ', logtime_format, ' ', time_since_start_s, 's', err_text, inpath, ' **' ])) if batch_print: print "".join(['**', err_text, inpath, ' **']) xmlBad2 = xmlBad2 + 1 # Rename problem files keepFile(outfileName, 2) elif fileNoRead: # Case cdscan reports no error, however file wasn't readable if '/data/cmip5/' in inpath: err_text = ' DATA PROBLEM 3 (read perms) indexing ' else: err_text = ' PROBLEM 3 (read perms) indexing ' writeToLog( logFile, "".join([ '** ', format(xmlBad3, "07d"), ' ', logtime_format, ' ', time_since_start_s, 's', err_text, inpath, ' **' ])) if batch_print: print "".join(['**', err_text, inpath, ' **']) xmlBad3 = xmlBad3 + 1 # Rename problem files keepFile(outfileName, 3) elif fileNoWrite: # Case cdscan reports no error, however file wasn't written if '/data/cmip5/' in inpath: err_text = ' DATA PROBLEM 4 (no outfile) indexing ' else: err_text = ' PROBLEM 4 (no outfile) indexing ' writeToLog( logFile, "".join([ '** ', format(xmlBad4, "07d"), ' ', logtime_format, ' ', time_since_start_s, 's', err_text, inpath, ' **' ])) if batch_print: print "".join(['**', err_text, inpath, ' **']) xmlBad4 = xmlBad4 + 1 # Rename problem files keepFile(outfileName, 4) elif fileNone: # Case cdscan reports no error, however file wasn't written if '/data/cmip5/' in inpath: err_text = ' DATA PROBLEM 5 (no infiles) indexing ' else: err_text = ' PROBLEM 5 (no infiles) indexing ' writeToLog( logFile, "".join([ '** ', format(xmlBad5, "07d"), ' ', logtime_format, ' ', time_since_start_s, 's', err_text, inpath, ' **' ])) if batch_print: print "".join(['**', err_text, inpath, ' **']) xmlBad5 = xmlBad5 + 1 # Rename problem files keepFile(outfileName, 5) else: writeToLog( logFile, "".join([ '** ', format(xmlGood, "07d"), ' ', logtime_format, ' ', time_since_start_s, 's success creating: ', outfileName, ' **' ])) xmlGood = xmlGood + 1 return [xmlBad1, xmlBad2, xmlBad3, xmlBad4, xmlBad5, xmlGood] # ; Non-parallel version of code
from durolib import fixVarUnits, globalAttWrite, writeToLog # ,trimModelList # %% Set current dirs workDur = "/p/user_pub/climate_work/durack1" workDir = os.path.join(workDur, "Shared/210128_PaperPlots_Rothigetal/") xmlPath = "/p/user_pub/xclim/" # '/data_crunchy_oceanonly/crunchy_work/cmip-dyn' # %% Generate log file timeNow = datetime.datetime.now() timeFormat = timeNow.strftime("%y%m%dT%H%M%S") dateNow = timeNow.strftime("%y%m%d") # dateNow = '210226' logFile = os.path.join(workDir, "_".join([timeFormat, "RothigData_log.txt"])) textToWrite = " ".join(["TIME:", timeFormat]) writeToLog(logFile, textToWrite) pypid = str(os.getpid()) # Returns calling python instance, so master also see os.getppid() - Parent writeToLog(logFile, " ".join(["MASTER PID:", pypid])) writeToLog(logFile, " ".join(["UV-CDAT:", sys.executable])) host_name = gethostname() print(" ".join(["HOSTNAME:", host_name])) writeToLog(logFile, " ".join(["HOSTNAME:", host_name])) print("----------") writeToLog(logFile, "----------") # %% Preallocate lists and fill fileLists = [] mipEra = "CMIP6" actExpPair = {} actExpPair["CMIP"] = {}
def logWrite(logfile,time_since_start,path_name,i1,data_outfiles,len_vars): outfile_count = len(data_outfiles) time_since_start_s = '%09.2f' % time_since_start print "".join([path_name.ljust(13),' scan complete.. ',format(i1,"1d").ljust(6),' paths total; ',str(outfile_count).ljust(6),' output files to be written (',format(len_vars,"1d").ljust(3),' vars sampled)']) writeToLog(logfile,"".join([time_since_start_s,' : ',path_name.ljust(13),' scan complete.. ',format(i1,"1d").ljust(6),' paths total; ',format(outfile_count,"1d").ljust(6),' output files to be written (',format(len_vars,"1d").ljust(3),' vars sampled)'])) return
# Set host information and directories host_name = gethostname() if host_name in {'crunchy.llnl.gov','oceanonly.llnl.gov'}: trim_host = replace(host_name,'.llnl.gov','') host_path = '/work/durack1/Shared/cmip3/' ; # crunchy 121129 cdat_path = '/usr/local/uvcdat/latest/bin/' else: print '** HOST UNKNOWN, aborting.. **' sys.exit() # Set logfile attributes time_now = datetime.datetime.now() time_format = time_now.strftime("%y%m%d_%H%M%S") logfile = os.path.join(host_path,"".join([time_format,'_make_cmip3_trendsAndClims-',experiment,'-',trim_host,'.log'])) writeToLog(logfile,"".join(['TIME: ',time_format,'\n'])) writeToLog(logfile,"".join(['HOSTNAME: ',host_name,'\n'])) # Get list of infiles (*.nc) and 3D (*.xml) # Get list of infiles (xml) if all_files: filelist1 = glob.glob("".join([host_path,'*/*/an/*/*.nc'])) filelist2 = glob.glob("".join([host_path,'*/*/an/*/*.xml'])) else: filelist1 = glob.glob("".join([host_path,experiment,'/*/an/*/*.nc'])) filelist2 = glob.glob("".join([host_path,experiment,'/*/an/*/*.xml'])) filelist = list(filelist1) filelist.extend(filelist2) ; filelist.sort() del(filelist1,filelist2) gc.collect()
pathin = '/work/cmip3' elif 'cmip5' in model_suite: host_path = '/work/durack1/Shared/cmip5/' pathin = '/work/cmip5' ; # set xml paths # Load drift dictionary cmip5_branch_time_dict = pickle.load(open(os.path.join(host_path,"".join([model_suite,'_branch_time_dict.pickle'])),'rb')) else: print '** HOST UNKNOWN, aborting.. **' sys.exit() # Set logfile attributes time_format = datetime.datetime.now().strftime("%y%m%d_%H%M%S") logfile = os.path.join(host_path,"".join([time_format,'_make_',model_suite,'_trendsAndClims-',experiment,'-',realm,'-',variable,'-',time_yrs,'-',trim_host,'.log'])) #logfile = os.path.join(host_path,'tmp',"".join([time_format,'_make_',model_suite,'_trendsAndClims-',experiment,'-',realm,'-',variable,'-',time_yrs,'-',trim_host,'.log'])) ; ## TEST ## # Create logfile writeToLog(logfile,"".join(['TIME: ',time_format])) writeToLog(logfile,"".join(['HOSTNAME: ',host_name])) del(time_format) ; gc.collect() #%% Get list of infiles (*.nc) and 3D (*.xml) if all_files and all_realms: print 'all_files and all_realms' filelist1 = glob.glob(os.path.join(host_path,'*/*/an/*/*.nc')) filelist2 = glob.glob(os.path.join(host_path,'*/*/an/*/*.xml')) elif all_realms: print 'all_realms' filelist1 = glob.glob(os.path.join(host_path,experiment,'*/an/*/*.nc')) filelist2 = glob.glob(os.path.join(host_path,experiment,'*/an/*/*.xml')) elif variable not in 'all': print ''.join(['variable: ',variable]) filelist1 = glob.glob(os.path.join(host_path,experiment,'*/an',variable,'*.nc'))
#modelSuite = 'cmip5' #experiment = 'historical' #experiment = 'rcp85' #modelSuite = 'cmip3' #experiment = '20c3m' #experiment = 'sresa2' #%% # Create logfile timeFormat = datetime.datetime.now().strftime("%y%m%d_%H%M%S") if os.getlogin() == 'durack1': logPath = '/export/durack1/git/Density_bining/' elif os.getlogin() == 'eguil': logPath = '/work/guilyardi/git/Density_bining/' logfile = os.path.join(logPath,"".join([timeFormat,'_drive_density-',modelSuite,'-',experiment,'-',gethostname().split('.')[0],'.log'])) ; # WORK MODE writeToLog(logfile,"".join(['TIME: ',timeFormat])) writeToLog(logfile,"".join(['HOSTNAME: ',gethostname()])) print "".join(['** Processing files from ',modelSuite,' for ',experiment,' **']) writeToLog(logfile,"".join(['** Processing files from ',modelSuite,' for ',experiment,' **'])) #%% # Set generic paths soPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/so'); thetaoPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/thetao'); fxPath = os.path.join('/work',modelSuite,'fx/fx/areacello'); # Validate paths if not os.path.exists(outPath): pass #os.makedirs(outPath) if not os.path.exists(soPath) or not os.path.exists(thetaoPath) or not os.path.exists(fxPath):
# Create logfile time_now = datetime.datetime.now() time_format = time_now.strftime("%y%m%d_%H%M%S") logfile = os.path.join( host_path, "".join([ time_format, '_make_', model_suite, '_an3D-', exp, '-', var, '-', trim_host, '.log' ])) # Log disk usage info for $machine:/work os.chdir('/work') cmd = 'df -h /work' o = os.popen(cmd).readlines() oj = "".join(o) # Open logfile to write writeToLog(logfile, "".join(['TIME: ', time_format, '\n'])) writeToLog(logfile, "".join(['HOSTNAME: ', host_name, '\n'])) writeToLog(logfile, "".join(['WORKDISK: ', oj])) # Set inputs indir = os.path.join(pathin, exp, 'ocn/mo', var) # *.xmls outdir = os.path.join(host_path, exp, 'ocn/an', var, 'ncs') # Get input xml files lst = os.listdir(indir) if reverse: lst.sort(reverse=True) else: lst.sort() # Loop over inputs
experiment = 'historical' outPath = '/work/guilyardi/Shared/data_density/test' #experiment = 'rcp85' #outPath = '/work/guilyardi/git/Density_bining/test_cmip5' #outPath = os.path.join('/work/durack1/Shared/data_density',datetime.datetime.now().strftime("%y%m%d")); #modelSuite = 'cmip3' #experiment = '20c3m' #experiment = 'sresa2' ''' # Create logfile timeFormat = datetime.datetime.now().strftime("%y%m%d_%H%M%S") logPath = '/export/durack1/git/Density_bining/' #logPath = '/work/guilyardi/git/Density_bining/' logfile = os.path.join(logPath,"".join([timeFormat,'_drive_surface-',modelSuite,'-',experiment,'-',gethostname().split('.')[0],'.log'])) ; # WORK MODE writeToLog(logfile,"".join(['TIME: ',timeFormat])) writeToLog(logfile,"".join(['HOSTNAME: ',gethostname()])) print "".join(['** Processing files from ',modelSuite,' for ',experiment,' **']) writeToLog(logfile,"".join(['** Processing files from ',modelSuite,' for ',experiment,' **'])) #%% # Set generic paths soPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/sos'); thetaoPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/tos'); hfdsPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/hfds'); wfoPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/wfo'); tauuPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/tauuo'); tauvPath = os.path.join('/work',modelSuite,experiment,'ocn/mo/tauvo'); fxPath = os.path.join('/work',modelSuite,'fx/fx/areacello'); # Validate paths
model_suite = 'cmip5' ; ##TEST_MODE## var = 'uo' ; ##TEST_MODE## exp = 'historical' ; ##TEST_MODE## ''' # Create logfile time_now = datetime.datetime.now() time_format = time_now.strftime("%y%m%d_%H%M%S") logfile = os.path.join(host_path,"".join([time_format,'_make_',model_suite,'_an3D-',exp,'-',var,'-',trim_host,'.log'])) # Log disk usage info for $machine:/work os.chdir('/work') cmd = 'df -h /work' o = os.popen(cmd).readlines() oj = "".join(o) # Open logfile to write writeToLog(logfile,"".join(['TIME: ',time_format,'\n'])) writeToLog(logfile,"".join(['HOSTNAME: ',host_name,'\n'])) writeToLog(logfile,"".join(['WORKDISK: ',oj])) # Set inputs indir = os.path.join(pathin,exp,'ocn/mo',var) # *.xmls outdir = os.path.join(host_path,exp,'ocn/an',var,'ncs') # Get input xml files lst = os.listdir(indir); if reverse: lst.sort(reverse=True); else: lst.sort() # Loop over inputs