def getscriptnamesregression(): # get a list of all folders in regression set flist = ryfiles.listFiles(pathToRegressionData, patterns='*', recurse=0, return_folders=1) ryscripts = [] for fli in flist: if not os.path.isfile(fli): ryscripts.append(fli) return ryscripts
def QueryDelete(recurse, dir, patn): thefiles = ryfiles.listFiles(dir, patn, recurse) if len(thefiles) > 0: for filename in thefiles: print(filename) if sys.version_info[0] < 3: instr = raw_input("Delete these files? (y/n)") else: instr = input("Delete these files? (y/n)") if instr == 'y': for filename in thefiles: os.remove(filename)
def runModtranAndCopy(root, research, pathToModtranBin, execname): """ Look for input files in directories, run modtran and copy results to dir. Finds all files below the root directory that matches the regex pattern in research. Then runs modtran on these files and write the tape5/6/7/8 back to where the input file was. Each input file must be in a separate directory, because the results are all written to files with the names 'tape5', etc. Args: | root ([string]): path to root dir containing the dirs with modtran input files. | research ([string]): regex to use when searching for input files ('.*.ltn' or 'tape5') | pathToModtranBin ([string]): path to modtran executable directory. | execname ([string]): modtran executable filename. Returns: | List of all the files processed. Raises: | No exception is raised. """ import os import shutil import subprocess import time import pyradi.ryfiles as ryfiles filepaths = ryfiles.listFiles(root, patterns=research, recurse=1, return_folders=0, useRegex=True) # print('**********************',root,research,filepaths) for filepath in filepaths: filename = os.path.basename(filepath) dirname = os.path.dirname(filepath) #get rid of clutter to make sure that our tape5 will be used for rfile in [ 'tape5', 'modin', 'modin.ontplt', 'mod5root.in', '.ezl20ck' ]: rpath = os.path.join(pathToModtranBin, rfile) if os.path.exists(rpath): os.remove(rpath) #copy our tape5 across tape5path = os.path.join(pathToModtranBin, 'tape5') shutil.copy2(filepath, tape5path) #run modtran on the tape5 file in its bin directory if os.path.exists(tape5path): p = subprocess.Popen(os.path.join(pathToModtranBin, execname), shell=True, stdout=None, stderr=None, cwd=pathToModtranBin) while p.poll() == None: time.sleep(0.5) # #copy the tape5/6/7 files back to appropriate directory for outname in ['tape5', 'tape6', 'tape7', 'tape8']: outpath = os.path.join(pathToModtranBin, outname) if os.path.exists(outpath): shutil.copy2(outpath, dirname)
import re import os.path as path import os import pyradi.ryfiles as ryfiles # rename YYYY_Book_somename.pdf to -somename-(YYYY).pdf filenames = ryfiles.listFiles('.','*.pdf',recurse=0) for fname in filenames: rtv = re.findall(r"(^\d{4})(_Book_)(.*)(\.pdf$)", fname) if len(rtv) > 0: name = re.sub(r"(\w)([A-Z])", r"\1 \2", rtv[0][2]) newname = f' - {name}-({rtv[0][0]}){rtv[0][3]}' print(fname, newname) if not path.exists(newname): os.rename(fname,newname) # print(name) print(newname) # print('--------------------------') else: print(f'*** repeat name: {newname}')
def runModtranAndCopy(root, research, pathToModtranBin, execname): """ Look for input files in directories, run modtran and copy results to dir. Finds all files below the root directory that matches the regex pattern in research. Then runs modtran on these files and write the tape5/6/7/8 back to where the input file was. Each input file must be in a separate directory, because the results are all written to files with the names 'tape5', etc. Args: | root ([string]): path to root dir containing the dirs with modtran input files. | research ([string]): regex to use when searching for input files ('.*.ltn' or 'tape5') | pathToModtranBin ([string]): path to modtran executable directory. | execname ([string]): modtran executable filename. Returns: | List of all the files processed. Raises: | No exception is raised. """ import os import shutil import subprocess import time import pyradi.ryfiles as ryfiles filepaths = ryfiles.listFiles(root, patterns=research, recurse=1, return_folders=0, useRegex=True) # print(len(filenames)) # print(filenames) for filepath in filepaths: filename = os.path.basename(filepath) dirname = os.path.dirname(filepath) # get rid of clutter to make sure that our tape5 will be used for rfile in ["tape5", "modin", "modin.ontplt", "mod5root.in", ".ezl20ck"]: rpath = os.path.join(pathToModtranBin, rfile) if os.path.exists(rpath): os.remove(rpath) # copy our tape5 across tape5path = os.path.join(pathToModtranBin, "tape5") shutil.copy2(filepath, tape5path) # run modtran on the tape5 file in its bin directory if os.path.exists(tape5path): p = subprocess.Popen( os.path.join(pathToModtranBin, execname), shell=True, stdout=None, stderr=None, cwd=pathToModtranBin ) while p.poll() == None: time.sleep(0.5) # #copy the tape5/6/7 files back to appropriate directory for outname in ["tape5", "tape6", "tape7", "tape8"]: outpath = os.path.join(pathToModtranBin, outname) if os.path.exists(outpath): shutil.copy2(outpath, dirname)
"""Markdown requires spaces after heading #, so insert space where required. If # is followed by anything other than a whitespace or a #, inset a space """ import pyradi.ryfiles as ryfiles import re filenames = ryfiles.listFiles('.', '*.md', 0, 0) # print(filenames) for filename in filenames: print(filename) # if '000-contents.md' in filename: linesOut = [] with open(filename, 'rt') as f: lines = f.readlines() for line in lines: line = re.sub(r'(#+)([^\s#])', r'\1 \2', line) linesOut.append(line) with open(filename, 'wt') as f: f.writelines(linesOut)
def calcEffective(ilines, alt, dir, specranges, dffilename): dirname = os.path.join('.', dir, 'elev', '{:.0f}'.format(alt)) lfiles = ryfiles.listFiles(dirname, patterns='tape7') # print(dirname) dfCols = [ 'Atmo', 'Altitude', 'Zenith', 'SpecBand', 'ToaWattTot', 'ToaWatt', 'BoaWattTot', 'BoaWatt', 'LpathWatt', 'ToaQTot', 'ToaQ', 'BoaQTot', 'BoaQ', 'LpathQ', 'effTauSun', 'effTau300' ] #open the excel file, read in, append to the dataframe and later save to the same file if os.path.exists(dffilename): df = pd.read_excel(dffilename) else: df = pd.DataFrame(columns=dfCols) for filename in lfiles: print(filename) elev = float(filename.split('\\')[3]) #read transmittance and path radiance from tape7 dataset = rymodtran.loadtape7( filename, ['FREQ', 'TOT_TRANS', 'TOA_SUN', 'TOTAL_RAD']) dataset[:, 2] *= 1e4 # convert from /cm2 to /m2 dataset[:, 3] *= 1e4 # but only do this once for data set for key in specranges: tape7 = dataset.copy() # select only lines in the spectral ranges select = np.all([ tape7[:, 0] >= 1e4 / specranges[key][1], tape7[:, 0] <= 1e4 / specranges[key][0] ], axis=0) # print(tape7[select]) tape7s = tape7[select] #now integrate the path and TOA radiances. now in W/(m2.sr.cm-1) TOAwatttot = np.trapz(tape7[:, 2], tape7[:, 0]) TOAwatt = np.trapz(tape7s[:, 2], tape7s[:, 0]) BOAwatttot = np.trapz(tape7[:, 1] * tape7[:, 2], tape7[:, 0]) BOAwatt = np.trapz(tape7s[:, 1] * tape7s[:, 2], tape7s[:, 0]) Lpathwatt = np.trapz(tape7s[:, 3], tape7s[:, 0]) # convert radiance terms to photon rates, must to this while it is still spectral # then integrate after conversion conv = tape7[:, 0] * const.h * const.c * 1e2 tape7[:, 2] /= conv tape7[:, 3] /= conv tape7s = tape7[select] #now integrate the path and TOA radiances. now in (q/s)/(m2.sr.cm-1) TOAqtot = np.trapz(tape7[:, 2], tape7[:, 0]) TOAq = np.trapz(tape7s[:, 2], tape7s[:, 0]) BOAqtot = np.trapz(tape7[:, 1] * tape7[:, 2], tape7[:, 0]) BOAq = np.trapz(tape7s[:, 1] * tape7s[:, 2], tape7s[:, 0]) Lpathq = np.trapz(tape7s[:, 3], tape7s[:, 0]) LSun = ryplanck.planck(tape7s[:, 0], 6000., 'en') L300 = ryplanck.planck(tape7s[:, 0], 300., 'en') effTauSun = np.trapz(LSun * tape7s[:, 1], tape7s[:, 0]) / np.trapz( LSun, tape7s[:, 0]) effTau300 = np.trapz(L300 * tape7s[:, 1], tape7s[:, 0]) / np.trapz( L300, tape7s[:, 0]) # print(elev,key, BOAwatttot, BOAwatt, TOAwatttot, TOAwatt, Lpathwatt) # print(elev,key, BOAqtot, BOAq, TOAqtot, TOAq, Lpathq) df = df.append( pd.DataFrame([[ dir, alt, elev, key, TOAwatttot, TOAwatt, BOAwatttot, BOAwatt, Lpathwatt, TOAqtot, TOAq, BOAqtot, BOAq, Lpathq, effTauSun, effTau300 ]], columns=dfCols)) ilines += 1 writer = pd.ExcelWriter(dffilename) df.to_excel(writer, 'Sheet1') pd.DataFrame(specranges).to_excel(writer, 'SpecRanges') writer.save() return (ilines)
def compareenvresults(envis, ryscripts, fout=None, similarity=0.01): """Compare files with the same names in each of environments """ fdict = {} fnames = {} for ryscript in ryscripts: # get the script name for this run script = os.path.basename(ryscript) fdict[script] = {} for envi in envis: fdict[script][envi] = {} # capture the files in the environment paths = ryfiles.listFiles(os.path.join(ryscript, envi), patterns='*', recurse=0, return_folders=0) for path in paths: fname = os.path.basename(path) fnames[fname] = 1 if os.path.isfile(path): fdict[script][envi][fname] = hash_file(path) else: fdict[script][envi][fname] = None # finished collecting, now compare sameHash = {} sameTxt = {} sameImg = {} diffEnv = {} for ryscript in ryscripts: script = os.path.basename(ryscript) for fname in fnames: hsh0 = None if script in fdict.keys(): if envis[0] in fdict[script].keys(): if fname in fdict[script][envis[0]].keys(): hsh0 = fdict[script][envis[0]][fname] hsh1 = None if script in fdict.keys(): if envis[1] in fdict[script].keys(): if fname in fdict[script][envis[1]].keys(): hsh1 = fdict[script][envis[1]][fname] if hsh0 is not None and hsh1 is not None: if hsh0 == hsh1: sameHash['{}/{}'.format(script, fname)] = 1 else: path0 = os.path.join(pathToRegressionData, script, envis[0], fname) path1 = os.path.join(pathToRegressionData, script, envis[1], fname) # do text compare if text file if not is_binary_string(open(path0, 'rb').read(1024)): if cmp_lines(path0, path1): sameTxt['{}/{}'.format(script, fname)] = 1 else: diffEnv['{}/{}'.format(script, fname)] = 1 else: # hashes and text content mismatch, these are probably bitmaps or binary files imgdiff = imagedifference(path0, path1) if imgdiff < similarity: sameImg['{}/{}'.format(script, fname)] = imgdiff else: print('{} {}'.format(path0, imgdiff)) diffEnv['{}/{}'.format(script, fname)] = 1 if fout is not None: fout.write('\n\nHash the same in both environments:\n') for key in sorted(list(sameHash.keys())): fout.write(' + {}\n'.format(key)) fout.write('\n\nText the same in both environments:\n') for key in sorted(list(sameTxt.keys())): fout.write(' + {}\n'.format(key)) fout.write( '\n\nImages are similar in both environments with normalised error less than {}:\n' .format(similarity)) for key in sorted(list(sameImg.keys())): fout.write(' + {} {:.5f}\n'.format(key, sameImg[key])) fout.write('\n\nDifferent between environments:\n') for key in sorted(list(diffEnv.keys())): fout.write(' - {} {:.5f}\n'.format(key, diffEnv[key])) return sameHash, sameTxt, diffEnv
def runscripts(ryscripts, fout=None, similarity=0.01): # run tests for each script and compare against its own ref set passed = {} failed = {} missing = {} for ryscript in ryscripts: # get the script name for this run script = os.path.basename(ryscript) #build a commandline string to execute and write stdout to file task = 'python {}.py >{}-{}.txt'.format(script, script, cenvi) print('\n{}'.format(task)) out = subprocess.check_output(task) with open('{}-{}.txt'.format(script, cenvi), 'wb') as foutlocal: foutlocal.write(out) for envi in envis: if envi in cenvi: # replace with True to cross check if envi in cenvi: strenvi = '' else: strenvi = ' in {}'.format(envi) #get the list of files in the regression folder targfolder = os.path.join(pathToRegressionData, script.split('.')[0], envi) print( 'Regression result files in folder: {}'.format(targfolder)) rlist = ryfiles.listFiles(targfolder, patterns='*.*', recurse=0, return_folders=0) for fregres in rlist: flocal = os.path.basename(fregres) if os.path.exists(flocal): hflocal = hash_file(flocal) if os.path.exists(fregres): hfregres = hash_file(fregres) if hflocal == hfregres: result = '+' passed['{}{}'.format(flocal, strenvi)] = 1 else: # test for text string compare if not is_binary_string( open(fregres, 'rb').read(1024)): if cmp_lines(fregres, flocal): passed['{}{}'.format(flocal, strenvi)] = 1 else: failed['{}{}'.format(flocal, strenvi)] = 1 else: # hashes and text content mismatch, these are probably bitmaps or binary files imgdiff = imagedifference(flocal, fregres) if imgdiff < similarity: result = '+' passed['{}{} -- {:.5f}'.format( flocal, strenvi, imgdiff)] = 1 else: result = '-' failed['{}{} -- {:.5f}'.format( flocal, strenvi, imgdiff)] = 1 # else: # result = '?' # missing['{}{}'.format(flocal,strenvi)] = 1 else: result = '?' missing['{}{}'.format(flocal, strenvi)] = 1 print(' {} {} vs {}'.format(result, fregres, flocal)) if fout is not None: fout.write('\n\nPassed:\n') for key in sorted(list(passed.keys())): fout.write(' + {}\n'.format(key)) fout.write('\n\nFailed:\n') for key in sorted(list(failed.keys())): fout.write(' - {}\n'.format(key)) fout.write('\n\nMissing:\n') for key in sorted(list(missing.keys())): fout.write(' ? {}\n'.format(key)) return passed, failed, missing
import pyradi.ryfiles as ryfiles import regex texfiles = ryfiles.listFiles('.', '*.tex') # print(texfiles) lensel = 20 dPairs = {} p = regex.compile(r'\\url\{(.+?)\}') for texfile in texfiles: with open(texfile, 'r') as fin: ifchanged = False filelines = [] lines = fin.readlines() for line in lines: urls = p.findall(line) if len(urls) > 0: for url in urls: # clean up the URL removing some chars tmpn = ryfiles.cleanFilename( url, removestring=" -_%:/,.\\[]<>*?#!'\"=&") # get rid of http(s) tmpn = tmpn.replace('https', '').replace('http', '') # make long enough string while len(tmpn) < lensel: tmpn = tmpn + tmpn #build cite key from start and end of string citename = tmpn[0:lensel] + tmpn[-lensel:] # append to the dict if url not in dPairs:
"""Markdown requires spaces after heading #, so insert space where required. If # is followed by anything other than a whitespace or a #, inset a space """ import pyradi.ryfiles as ryfiles import re filenames = ryfiles.listFiles('.','*.md', 0,0) # print(filenames) for filename in filenames: print(filename) # if '000-contents.md' in filename: linesOut = [] with open(filename, 'rt') as f: lines = f.readlines() for line in lines: line = re.sub(r'(#+)([^\s#])', r'\1 \2', line) linesOut.append(line) with open(filename, 'wt') as f: f.writelines(linesOut)