def getscriptnamesregression():
# get a list of all folders in regression set
flist = ryfiles.listFiles(pathToRegressionData,
patterns='*',
recurse=0,
return_folders=1)
ryscripts = []
for fli in flist:
if not os.path.isfile(fli):
ryscripts.append(fli)
return ryscripts
def QueryDelete(recurse, dir, patn):
thefiles = ryfiles.listFiles(dir, patn, recurse)
if len(thefiles) > 0:
for filename in thefiles:
print(filename)
if sys.version_info[0] < 3:
instr = raw_input("Delete these files? (y/n)")
else:
instr = input("Delete these files? (y/n)")
if instr == 'y':
for filename in thefiles:
os.remove(filename)
def runModtranAndCopy(root, research, pathToModtranBin, execname):
"""
Look for input files in directories, run modtran and copy results to dir.
Finds all files below the root directory that matches the
regex pattern in research. Then runs modtran on these files
and write the tape5/6/7/8 back to where the input file was.
Each input file must be in a separate directory, because the results are
all written to files with the names 'tape5', etc.
Args:
| root ([string]): path to root dir containing the dirs with modtran input files.
| research ([string]): regex to use when searching for input files ('.*.ltn' or 'tape5')
| pathToModtranBin ([string]): path to modtran executable directory.
| execname ([string]): modtran executable filename.
Returns:
| List of all the files processed.
Raises:
| No exception is raised.
"""
import os
import shutil
import subprocess
import time
import pyradi.ryfiles as ryfiles
filepaths = ryfiles.listFiles(root,
patterns=research,
recurse=1,
return_folders=0,
useRegex=True)
# print('**********************',root,research,filepaths)
for filepath in filepaths:
filename = os.path.basename(filepath)
dirname = os.path.dirname(filepath)
#get rid of clutter to make sure that our tape5 will be used
for rfile in [
'tape5', 'modin', 'modin.ontplt', 'mod5root.in', '.ezl20ck'
]:
rpath = os.path.join(pathToModtranBin, rfile)
if os.path.exists(rpath):
os.remove(rpath)
#copy our tape5 across
tape5path = os.path.join(pathToModtranBin, 'tape5')
shutil.copy2(filepath, tape5path)
#run modtran on the tape5 file in its bin directory
if os.path.exists(tape5path):
p = subprocess.Popen(os.path.join(pathToModtranBin, execname),
shell=True,
stdout=None,
stderr=None,
cwd=pathToModtranBin)
while p.poll() == None:
time.sleep(0.5)
# #copy the tape5/6/7 files back to appropriate directory
for outname in ['tape5', 'tape6', 'tape7', 'tape8']:
outpath = os.path.join(pathToModtranBin, outname)
if os.path.exists(outpath):
shutil.copy2(outpath, dirname)
import re
import os.path as path
import os
import pyradi.ryfiles as ryfiles
# rename YYYY_Book_somename.pdf to -somename-(YYYY).pdf
filenames = ryfiles.listFiles('.','*.pdf',recurse=0)
for fname in filenames:
rtv = re.findall(r"(^\d{4})(_Book_)(.*)(\.pdf$)", fname)
if len(rtv) > 0:
name = re.sub(r"(\w)([A-Z])", r"\1 \2", rtv[0][2])
newname = f' - {name}-({rtv[0][0]}){rtv[0][3]}'
print(fname, newname)
if not path.exists(newname):
os.rename(fname,newname)
# print(name)
print(newname)
# print('--------------------------')
else:
print(f'*** repeat name: {newname}')
def runModtranAndCopy(root, research, pathToModtranBin, execname):
"""
Look for input files in directories, run modtran and copy results to dir.
Finds all files below the root directory that matches the
regex pattern in research. Then runs modtran on these files
and write the tape5/6/7/8 back to where the input file was.
Each input file must be in a separate directory, because the results are
all written to files with the names 'tape5', etc.
Args:
| root ([string]): path to root dir containing the dirs with modtran input files.
| research ([string]): regex to use when searching for input files ('.*.ltn' or 'tape5')
| pathToModtranBin ([string]): path to modtran executable directory.
| execname ([string]): modtran executable filename.
Returns:
| List of all the files processed.
Raises:
| No exception is raised.
"""
import os
import shutil
import subprocess
import time
import pyradi.ryfiles as ryfiles
filepaths = ryfiles.listFiles(root, patterns=research, recurse=1, return_folders=0, useRegex=True)
# print(len(filenames))
# print(filenames)
for filepath in filepaths:
filename = os.path.basename(filepath)
dirname = os.path.dirname(filepath)
# get rid of clutter to make sure that our tape5 will be used
for rfile in ["tape5", "modin", "modin.ontplt", "mod5root.in", ".ezl20ck"]:
rpath = os.path.join(pathToModtranBin, rfile)
if os.path.exists(rpath):
os.remove(rpath)
# copy our tape5 across
tape5path = os.path.join(pathToModtranBin, "tape5")
shutil.copy2(filepath, tape5path)
# run modtran on the tape5 file in its bin directory
if os.path.exists(tape5path):
p = subprocess.Popen(
os.path.join(pathToModtranBin, execname), shell=True, stdout=None, stderr=None, cwd=pathToModtranBin
)
while p.poll() == None:
time.sleep(0.5)
# #copy the tape5/6/7 files back to appropriate directory
for outname in ["tape5", "tape6", "tape7", "tape8"]:
outpath = os.path.join(pathToModtranBin, outname)
if os.path.exists(outpath):
shutil.copy2(outpath, dirname)
"""Markdown requires spaces after heading #, so insert space where required.
If # is followed by anything other than a whitespace or a #, inset a space
"""
import pyradi.ryfiles as ryfiles
import re
filenames = ryfiles.listFiles('.', '*.md', 0, 0)
# print(filenames)
for filename in filenames:
print(filename)
# if '000-contents.md' in filename:
linesOut = []
with open(filename, 'rt') as f:
lines = f.readlines()
for line in lines:
line = re.sub(r'(#+)([^\s#])', r'\1 \2', line)
linesOut.append(line)
with open(filename, 'wt') as f:
f.writelines(linesOut)
def calcEffective(ilines, alt, dir, specranges, dffilename):
dirname = os.path.join('.', dir, 'elev', '{:.0f}'.format(alt))
lfiles = ryfiles.listFiles(dirname, patterns='tape7')
# print(dirname)
dfCols = [
'Atmo', 'Altitude', 'Zenith', 'SpecBand', 'ToaWattTot', 'ToaWatt',
'BoaWattTot', 'BoaWatt', 'LpathWatt', 'ToaQTot', 'ToaQ', 'BoaQTot',
'BoaQ', 'LpathQ', 'effTauSun', 'effTau300'
]
#open the excel file, read in, append to the dataframe and later save to the same file
if os.path.exists(dffilename):
df = pd.read_excel(dffilename)
else:
df = pd.DataFrame(columns=dfCols)
for filename in lfiles:
print(filename)
elev = float(filename.split('\\')[3])
#read transmittance and path radiance from tape7
dataset = rymodtran.loadtape7(
filename, ['FREQ', 'TOT_TRANS', 'TOA_SUN', 'TOTAL_RAD'])
dataset[:, 2] *= 1e4 # convert from /cm2 to /m2
dataset[:, 3] *= 1e4 # but only do this once for data set
for key in specranges:
tape7 = dataset.copy()
# select only lines in the spectral ranges
select = np.all([
tape7[:, 0] >= 1e4 / specranges[key][1],
tape7[:, 0] <= 1e4 / specranges[key][0]
],
axis=0)
# print(tape7[select])
tape7s = tape7[select]
#now integrate the path and TOA radiances. now in W/(m2.sr.cm-1)
TOAwatttot = np.trapz(tape7[:, 2], tape7[:, 0])
TOAwatt = np.trapz(tape7s[:, 2], tape7s[:, 0])
BOAwatttot = np.trapz(tape7[:, 1] * tape7[:, 2], tape7[:, 0])
BOAwatt = np.trapz(tape7s[:, 1] * tape7s[:, 2], tape7s[:, 0])
Lpathwatt = np.trapz(tape7s[:, 3], tape7s[:, 0])
# convert radiance terms to photon rates, must to this while it is still spectral
# then integrate after conversion
conv = tape7[:, 0] * const.h * const.c * 1e2
tape7[:, 2] /= conv
tape7[:, 3] /= conv
tape7s = tape7[select]
#now integrate the path and TOA radiances. now in (q/s)/(m2.sr.cm-1)
TOAqtot = np.trapz(tape7[:, 2], tape7[:, 0])
TOAq = np.trapz(tape7s[:, 2], tape7s[:, 0])
BOAqtot = np.trapz(tape7[:, 1] * tape7[:, 2], tape7[:, 0])
BOAq = np.trapz(tape7s[:, 1] * tape7s[:, 2], tape7s[:, 0])
Lpathq = np.trapz(tape7s[:, 3], tape7s[:, 0])
LSun = ryplanck.planck(tape7s[:, 0], 6000., 'en')
L300 = ryplanck.planck(tape7s[:, 0], 300., 'en')
effTauSun = np.trapz(LSun * tape7s[:, 1], tape7s[:, 0]) / np.trapz(
LSun, tape7s[:, 0])
effTau300 = np.trapz(L300 * tape7s[:, 1], tape7s[:, 0]) / np.trapz(
L300, tape7s[:, 0])
# print(elev,key, BOAwatttot, BOAwatt, TOAwatttot, TOAwatt, Lpathwatt)
# print(elev,key, BOAqtot, BOAq, TOAqtot, TOAq, Lpathq)
df = df.append(
pd.DataFrame([[
dir, alt, elev, key, TOAwatttot, TOAwatt, BOAwatttot,
BOAwatt, Lpathwatt, TOAqtot, TOAq, BOAqtot, BOAq, Lpathq,
effTauSun, effTau300
]],
columns=dfCols))
ilines += 1
writer = pd.ExcelWriter(dffilename)
df.to_excel(writer, 'Sheet1')
pd.DataFrame(specranges).to_excel(writer, 'SpecRanges')
writer.save()
return (ilines)
def compareenvresults(envis, ryscripts, fout=None, similarity=0.01):
"""Compare files with the same names in each of environments
"""
fdict = {}
fnames = {}
for ryscript in ryscripts:
# get the script name for this run
script = os.path.basename(ryscript)
fdict[script] = {}
for envi in envis:
fdict[script][envi] = {}
# capture the files in the environment
paths = ryfiles.listFiles(os.path.join(ryscript, envi),
patterns='*',
recurse=0,
return_folders=0)
for path in paths:
fname = os.path.basename(path)
fnames[fname] = 1
if os.path.isfile(path):
fdict[script][envi][fname] = hash_file(path)
else:
fdict[script][envi][fname] = None
# finished collecting, now compare
sameHash = {}
sameTxt = {}
sameImg = {}
diffEnv = {}
for ryscript in ryscripts:
script = os.path.basename(ryscript)
for fname in fnames:
hsh0 = None
if script in fdict.keys():
if envis[0] in fdict[script].keys():
if fname in fdict[script][envis[0]].keys():
hsh0 = fdict[script][envis[0]][fname]
hsh1 = None
if script in fdict.keys():
if envis[1] in fdict[script].keys():
if fname in fdict[script][envis[1]].keys():
hsh1 = fdict[script][envis[1]][fname]
if hsh0 is not None and hsh1 is not None:
if hsh0 == hsh1:
sameHash['{}/{}'.format(script, fname)] = 1
else:
path0 = os.path.join(pathToRegressionData, script,
envis[0], fname)
path1 = os.path.join(pathToRegressionData, script,
envis[1], fname)
# do text compare if text file
if not is_binary_string(open(path0, 'rb').read(1024)):
if cmp_lines(path0, path1):
sameTxt['{}/{}'.format(script, fname)] = 1
else:
diffEnv['{}/{}'.format(script, fname)] = 1
else:
# hashes and text content mismatch, these are probably bitmaps or binary files
imgdiff = imagedifference(path0, path1)
if imgdiff < similarity:
sameImg['{}/{}'.format(script, fname)] = imgdiff
else:
print('{} {}'.format(path0, imgdiff))
diffEnv['{}/{}'.format(script, fname)] = 1
if fout is not None:
fout.write('\n\nHash the same in both environments:\n')
for key in sorted(list(sameHash.keys())):
fout.write(' + {}\n'.format(key))
fout.write('\n\nText the same in both environments:\n')
for key in sorted(list(sameTxt.keys())):
fout.write(' + {}\n'.format(key))
fout.write(
'\n\nImages are similar in both environments with normalised error less than {}:\n'
.format(similarity))
for key in sorted(list(sameImg.keys())):
fout.write(' + {} {:.5f}\n'.format(key, sameImg[key]))
fout.write('\n\nDifferent between environments:\n')
for key in sorted(list(diffEnv.keys())):
fout.write(' - {} {:.5f}\n'.format(key, diffEnv[key]))
return sameHash, sameTxt, diffEnv
def runscripts(ryscripts, fout=None, similarity=0.01):
# run tests for each script and compare against its own ref set
passed = {}
failed = {}
missing = {}
for ryscript in ryscripts:
# get the script name for this run
script = os.path.basename(ryscript)
#build a commandline string to execute and write stdout to file
task = 'python {}.py >{}-{}.txt'.format(script, script, cenvi)
print('\n{}'.format(task))
out = subprocess.check_output(task)
with open('{}-{}.txt'.format(script, cenvi), 'wb') as foutlocal:
foutlocal.write(out)
for envi in envis:
if envi in cenvi: # replace with True to cross check
if envi in cenvi:
strenvi = ''
else:
strenvi = ' in {}'.format(envi)
#get the list of files in the regression folder
targfolder = os.path.join(pathToRegressionData,
script.split('.')[0], envi)
print(
'Regression result files in folder: {}'.format(targfolder))
rlist = ryfiles.listFiles(targfolder,
patterns='*.*',
recurse=0,
return_folders=0)
for fregres in rlist:
flocal = os.path.basename(fregres)
if os.path.exists(flocal):
hflocal = hash_file(flocal)
if os.path.exists(fregres):
hfregres = hash_file(fregres)
if hflocal == hfregres:
result = '+'
passed['{}{}'.format(flocal, strenvi)] = 1
else:
# test for text string compare
if not is_binary_string(
open(fregres, 'rb').read(1024)):
if cmp_lines(fregres, flocal):
passed['{}{}'.format(flocal,
strenvi)] = 1
else:
failed['{}{}'.format(flocal,
strenvi)] = 1
else:
# hashes and text content mismatch, these are probably bitmaps or binary files
imgdiff = imagedifference(flocal, fregres)
if imgdiff < similarity:
result = '+'
passed['{}{} -- {:.5f}'.format(
flocal, strenvi, imgdiff)] = 1
else:
result = '-'
failed['{}{} -- {:.5f}'.format(
flocal, strenvi, imgdiff)] = 1
# else:
# result = '?'
# missing['{}{}'.format(flocal,strenvi)] = 1
else:
result = '?'
missing['{}{}'.format(flocal, strenvi)] = 1
print(' {} {} vs {}'.format(result, fregres, flocal))
if fout is not None:
fout.write('\n\nPassed:\n')
for key in sorted(list(passed.keys())):
fout.write(' + {}\n'.format(key))
fout.write('\n\nFailed:\n')
for key in sorted(list(failed.keys())):
fout.write(' - {}\n'.format(key))
fout.write('\n\nMissing:\n')
for key in sorted(list(missing.keys())):
fout.write(' ? {}\n'.format(key))
return passed, failed, missing
import pyradi.ryfiles as ryfiles
import regex
texfiles = ryfiles.listFiles('.', '*.tex')
# print(texfiles)
lensel = 20
dPairs = {}
p = regex.compile(r'\\url\{(.+?)\}')
for texfile in texfiles:
with open(texfile, 'r') as fin:
ifchanged = False
filelines = []
lines = fin.readlines()
for line in lines:
urls = p.findall(line)
if len(urls) > 0:
for url in urls:
# clean up the URL removing some chars
tmpn = ryfiles.cleanFilename(
url, removestring=" -_%:/,.\\[]<>*?#!'\"=&")
# get rid of http(s)
tmpn = tmpn.replace('https', '').replace('http', '')
# make long enough string
while len(tmpn) < lensel:
tmpn = tmpn + tmpn
#build cite key from start and end of string
citename = tmpn[0:lensel] + tmpn[-lensel:]
# append to the dict
if url not in dPairs:
"""Markdown requires spaces after heading #, so insert space where required.
If # is followed by anything other than a whitespace or a #, inset a space
"""
import pyradi.ryfiles as ryfiles
import re
filenames = ryfiles.listFiles('.','*.md', 0,0)
# print(filenames)
for filename in filenames:
print(filename)
# if '000-contents.md' in filename:
linesOut = []
with open(filename, 'rt') as f:
lines = f.readlines()
for line in lines:
line = re.sub(r'(#+)([^\s#])', r'\1 \2', line)
linesOut.append(line)
with open(filename, 'wt') as f:
f.writelines(linesOut)
