def links(self):
debug(self._label + 'create additional symlinks')
src_install = pjoin(Settings.CONTROL_PATH,
self._package['package'] + '.links')
if not pexists(src_install):
return
try:
lineno = 0
with open(src_install) as fin:
for line in fin:
lineno += 1
src, dst = line.strip().split(' ', 1)
dst = dst.strip()
if dst.startswith('/'):
dst = '.' + dst
dst = pjoin(self._env['QPKG_DEST_DATA'], dst)
if dst.endswith('/'):
if not pexists(dst):
makedirs(dst)
dst = pjoin(dst, pbasename(src))
else:
if not pexists(pdirname(dst)):
makedirs(pdirname(dst))
symlink(src, dst)
except ValueError:
raise FileSyntaxError(src_install, lineno, line)
def links(self):
debug(self._label + "create additional symlinks")
src_install = pjoin(Settings.CONTROL_PATH, self._package["package"] + ".links")
if not pexists(src_install):
return
try:
lineno = 0
with open(src_install) as fin:
for line in fin:
lineno += 1
src, dst = line.strip().split(" ", 1)
dst = dst.strip()
if dst.startswith("/"):
dst = "." + dst
dst = pjoin(self._env["QPKG_DEST_DATA"], dst)
if dst.endswith("/"):
if not pexists(dst):
makedirs(dst)
dst = pjoin(dst, pbasename(src))
else:
if not pexists(pdirname(dst)):
makedirs(pdirname(dst))
symlink(src, dst)
except ValueError:
raise FileSyntaxError(src_install, lineno, line)
def generate(env):
"""Add Builders and construction variables for swig to an Environment."""
import numpy.f2py
d = pdirname(numpy.f2py.__file__)
f2pyac = SCons.Action.Action(pyf2c, '$F2PYCOMSTR')
c_file, cxx_file = SCons.Tool.createCFileBuilders(env)
c_file.add_action('.pyf', SCons.Action.Action(f2pyac))
c_file.add_emitter('.pyf', F2pyEmitter)
env['F2PYOPTIONS'] = SCons.Util.CLVar('--quiet')
env['F2PYBUILDDIR'] = ''
env['F2PYINCLUDEDIR'] = pjoin(d, 'src')
if not ("F2PY_NOT_ADD_INCLUDEDIR" in env
and env["F2PY_NOT_ADD_INCLUDEDIR"]):
env.Prepend(CPPPATH=env["F2PYINCLUDEDIR"])
if 'F2PYCOMSTR' not in env:
env['F2PYCOMSTR'] = "f2py: generating $TARGET from $SOURCE"
# XXX: adding a scanner using c_file.add_scanner does not work...
expr = '(<)include_file=(\S+)>'
scanner = SCons.Scanner.ClassicCPP("F2PYScan", ".pyf", "F2PYPATH", expr)
env.Append(SCANNERS=scanner)
env['BUILDERS']['F2py'] = SCons.Builder.Builder(action=f2pyac,
emitter=[F2pyEmitter])
def generate_linux(env):
icc = WhereIs('icc')
if not icc:
warnings.warn("icc not found")
topdir = normpath(pjoin(pdirname(icc), os.pardir, os.pardir))
abi = get_abi(env)
return old_generate(env, abi=abi, topdir=topdir)
def NumpyPythonExtension(env, *args, **kw):
"""this builder is the same than pythonextension, except for the fact that
it put the target into where distutils expects it."""
if kw.has_key('NUMPYCPPPATH'):
npy_cpppath = kw['NUMPYCPPPATH']
else:
npy_cpppath = env['NUMPYCPPPATH']
if kw.has_key('PYEXTCPPPATH'):
py_cpppath = kw['PYEXTCPPPATH']
else:
py_cpppath = env['PYEXTCPPPATH']
oldval = copy.copy(py_cpppath)
try:
kw['PYEXTCPPPATH'] = npy_cpppath + py_cpppath
lib = env.PythonExtension(*args, **kw)
finally:
kw['PYEXTCPPPATH'] = oldval
# XXX: hack so that if the install builder source contains a directory, the
# directory is aknowledged, e.g. installing 'bin/foo' into 'bar' should
# give 'bar/bin/foo', whereas Install('bar', 'bin/foo') install 'foo' into
# 'bar'
inst_lib = []
for t in lib:
d = pdirname(str(t))
if d:
install_dir = pjoin(env.subst('$distutils_installdir'), d)
else:
install_dir = env.subst('$distutils_installdir')
inst_lib.extend(env.Install(install_dir, t))
return lib, inst_lib
def test_opt_file(self):
res = results()
fname = pjoin(self.fbase, 'data', 'dtu10mw.opt')
df = res.load_operation(fname)
tmp = [
5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
23, 24, 25
]
np.testing.assert_allclose(tmp, df['windspeed'].values)
self.assertEqual(df.values.shape, (21, 3))
fname = pjoin(pdirname(__file__), 'data', 'kb6.opt')
df = res.load_operation(fname)
tmp = [
4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25
]
np.testing.assert_allclose(tmp, df['windspeed'].values)
tmp = [
7.20147212792062, 14.1560777401151, 24.5923689569214,
39.2304595043255, 58.7985344647829, 83.2307972895692,
106.401798353616, 106.400099017382, 106.403429471703,
106.3986284106, 106.39861469995, 106.393720086642,
106.404364646693, 106.401389916882, 106.4047949236,
106.398523413816, 106.403149136568, 106.399813867904,
106.424042832599, 106.400584861663, 106.43349513828,
106.469433544515
]
np.testing.assert_allclose(tmp, df['P_aero'].values)
self.assertEqual(df.values.shape, (22, 5))
def generate_linux(env):
icc = WhereIs('icc')
if not icc:
warnings.warn("icc not found")
topdir = normpath(pjoin(pdirname(icc), os.pardir, os.pardir))
abi = get_abi(env)
return old_generate(env, abi=abi, topdir=topdir)
def pyf2c(target, source, env):
import numpy.f2py
import shutil
# We need filenames from source/target for path handling
target_file_names = [str(i) for i in target]
source_file_names = [str(i) for i in source]
# Get source files necessary for f2py generated modules
d = os.path.dirname(numpy.f2py.__file__)
source_c = pjoin(d, 'src', FOBJECT_FILE)
# Copy source files for f2py generated modules in the build dir
build_dir = pdirname(target_file_names[0])
# XXX: blah
if build_dir == '':
build_dir = '.'
try:
cpi = mangle_fortranobject(target_file_names[0], FOBJECT_FILE)
shutil.copy(source_c, pjoin(build_dir, cpi))
except IOError, e:
msg = "Error while copying fortran source files (error was %s)" % str(e)
raise IOError(msg)
def generate(env):
"""Add Builders and construction variables for swig to an Environment."""
import numpy.f2py
d = pdirname(numpy.f2py.__file__)
f2pyac = SCons.Action.Action(pyf2c, '$F2PYCOMSTR')
c_file, cxx_file = SCons.Tool.createCFileBuilders(env)
c_file.add_action('.pyf', SCons.Action.Action(f2pyac))
c_file.add_emitter('.pyf', F2pyEmitter)
env['F2PYOPTIONS'] = SCons.Util.CLVar('--quiet')
env['F2PYBUILDDIR'] = ''
env['F2PYINCLUDEDIR'] = pjoin(d, 'src')
if not (env.has_key("F2PY_NOT_ADD_INCLUDEDIR") and env["F2PY_NOT_ADD_INCLUDEDIR"]):
env.Prepend(CPPPATH = env["F2PYINCLUDEDIR"])
if not env.has_key('F2PYCOMSTR'):
env['F2PYCOMSTR'] = "f2py: generating $TARGET from $SOURCE"
# XXX: adding a scanner using c_file.add_scanner does not work...
expr = '(<)include_file=(\S+)>'
scanner = SCons.Scanner.ClassicCPP("F2PYScan", ".pyf", "F2PYPATH", expr)
env.Append(SCANNERS = scanner)
env['BUILDERS']['F2py'] = SCons.Builder.Builder(action = f2pyac,
emitter = [F2pyEmitter])
def pyf2c(target, source, env):
import numpy.f2py
import shutil
# We need filenames from source/target for path handling
target_file_names = [str(i) for i in target]
source_file_names = [str(i) for i in source]
# Get source files necessary for f2py generated modules
d = os.path.dirname(numpy.f2py.__file__)
source_c = pjoin(d, 'src', FOBJECT_FILE)
# Copy source files for f2py generated modules in the build dir
build_dir = pdirname(target_file_names[0])
# XXX: blah
if build_dir == '':
build_dir = '.'
try:
cpi = mangle_fortranobject(target_file_names[0], FOBJECT_FILE)
shutil.copy(source_c, pjoin(build_dir, cpi))
except IOError, e:
msg = "Error while copying fortran source files (error was %s)" % str(
e)
raise IOError(msg)
def NumpyPythonExtension(env, *args, **kw):
"""this builder is the same than pythonextension, except for the fact that
it put the target into where distutils expects it."""
if kw.has_key('NUMPYCPPPATH'):
npy_cpppath = kw['NUMPYCPPPATH']
else:
npy_cpppath = env['NUMPYCPPPATH']
if kw.has_key('PYEXTCPPPATH'):
py_cpppath = kw['PYEXTCPPPATH']
else:
py_cpppath = env['PYEXTCPPPATH']
oldval = copy.copy(py_cpppath)
try:
kw['PYEXTCPPPATH'] = npy_cpppath + py_cpppath
lib = env.PythonExtension(*args, **kw)
finally:
kw['PYEXTCPPPATH'] = oldval
# XXX: hack so that if the install builder source contains a directory, the
# directory is aknowledged, e.g. installing 'bin/foo' into 'bar' should
# give 'bar/bin/foo', whereas Install('bar', 'bin/foo') install 'foo' into
# 'bar'
inst_lib = []
for t in lib:
d = pdirname(str(t))
if d:
install_dir = pjoin(env.subst('$distutils_installdir'), d)
else:
install_dir = env.subst('$distutils_installdir')
inst_lib.extend(env.Install(install_dir, t))
return lib, inst_lib
def get_distutils_libdir(cmd, sconscript_path):
"""Returns the path where distutils install libraries, relatively to the
scons build directory."""
from numscons import get_scons_build_dir
scdir = pjoin(get_scons_build_dir(), pdirname(sconscript_path))
n = scdir.count(os.sep)
return pjoin(os.sep.join([os.pardir for i in range(n + 1)]), cmd.build_lib)
def get_distutils_libdir(cmd, sconscript_path):
"""Returns the path where distutils install libraries, relatively to the
scons build directory."""
from numscons import get_scons_build_dir
scdir = pjoin(get_scons_build_dir(), pdirname(sconscript_path))
n = scdir.count(os.sep)
return pjoin(os.sep.join([os.pardir for i in range(n + 1)]), cmd.build_lib)
def get_cxx_tool_path(compiler):
"""Given a distutils.ccompiler.CCompiler class, returns the path of the
toolset related to C compilation."""
fullpath_exec = find_executable(get_cxxcompiler_executable(compiler))
if fullpath_exec:
fullpath = pdirname(fullpath_exec)
else:
raise DistutilsSetupError("Could not find compiler executable info for scons")
return fullpath
def get_cxx_tool_path(compiler):
"""Given a distutils.ccompiler.CCompiler class, returns the path of the
toolset related to C compilation."""
fullpath_exec = find_executable(get_cxxcompiler_executable(compiler))
if fullpath_exec:
fullpath = pdirname(fullpath_exec)
else:
raise DistutilsSetupError(
"Could not find compiler executable info for scons")
return fullpath
def generate_api_emitter(target, source, env):
"""Returns the list of targets generated by the code generator for array
api and ufunc api."""
base, ext = split_ext(str(target[0]))
dir = pdirname(base)
ba = pbasename(base)
h = pjoin(dir, "__" + ba + ".h")
c = pjoin(dir, "__" + ba + ".c")
txt = base + ".txt"
# print h, c, txt
t = [h, c, txt]
return (t, source)
def generate_api_emitter(target, source, env):
"""Returns the list of targets generated by the code generator for array
api and ufunc api."""
base, ext = split_ext(str(target[0]))
dir = pdirname(base)
ba = pbasename(base)
h = pjoin(dir, '__' + ba + '.h')
c = pjoin(dir, '__' + ba + '.c')
txt = base + '.txt'
#print h, c, txt
t = [h, c, txt]
return (t, source)
def pyf2c(target, source, env):
import numpy.f2py
import shutil
# We need filenames from source/target for path handling
target_file_names = [str(i) for i in target]
source_file_names = [str(i) for i in source]
# Get source files necessary for f2py generated modules
d = os.path.dirname(numpy.f2py.__file__)
source_c = pjoin(d, 'src', FOBJECT_FILE)
# Copy source files for f2py generated modules in the build dir
build_dir = pdirname(target_file_names[0])
# XXX: blah
if build_dir == '':
build_dir = '.'
try:
cpi = mangle_fortranobject(target_file_names[0], FOBJECT_FILE)
shutil.copy(source_c, pjoin(build_dir, cpi))
except IOError as e:
msg = "Error while copying fortran source files (error was %s)" % str(
e)
raise IOError(msg)
basename = os.path.basename(str(target[0]).split('module')[0])
# XXX: handle F2PYOPTIONS being a string instead of a list
if is_pyf(source_file_names[0]):
# XXX: scons has a way to force buidler to only use one source file
if len(source_file_names) > 1:
raise NotImplementedError("FIXME: multiple source files")
wrapper = pjoin(build_dir, FWRAP_TEMPLATE % basename)
cmd = env['F2PYOPTIONS'] + \
[source_file_names[0], '--build-dir', build_dir]
st = f2py_cmd_exec(cmd)
if not os.path.exists(wrapper):
f = open(wrapper, 'w')
f.close()
else:
cmd = env['F2PYOPTIONS'] + source_file_names + \
['--build-dir', build_dir]
# fortran files, we need to give the module name
cmd.extend(['--lower', '-m', basename])
st = f2py_cmd_exec(cmd)
return 0
def get_numpy_include_dirs(sconscript_path):
"""Return include dirs for numpy.
The paths are relatively to the setup.py script path."""
from numscons import get_scons_build_dir
scdir = pjoin(get_scons_build_dir(), pdirname(sconscript_path))
n = scdir.count(os.sep)
dirs = _incdir()
rdirs = []
for d in dirs:
rdirs.append(pjoin(os.sep.join([os.pardir for i in range(n+1)]), d))
return rdirs
def get_numpy_include_dirs(sconscript_path):
"""Return include dirs for numpy.
The paths are relatively to the setup.py script path."""
from numscons import get_scons_build_dir
scdir = pjoin(get_scons_build_dir(), pdirname(sconscript_path))
n = scdir.count(os.sep)
dirs = _incdir()
rdirs = []
for d in dirs:
rdirs.append(pjoin(os.sep.join([os.pardir for i in range(n + 1)]), d))
return rdirs
def createAllCredits(path):
fm = FolderManager()
subs, imp = findAllSubmissionFolders(path)
total = []
extra = []
curParent = path
sheetNr = ''
while True:
tmp = pdirname(curParent)
if tmp == curParent:
raise ValueError("U can't use this script here!")
curParent = tmp
if rcompile('Blatt_\d\d').match(pbasename(curParent)):
sheetNr = pbasename(curParent).split('_')[1]
break
for subpath in subs:
for tup in createCreditsText(subpath):
total.append((tup[0], [('%.2f' % x).zfill(5) for x in tup[1]]))
for subpath in imp:
for tup in createCreditsText(subpath):
searchRes = fm.findStudentByName(tup[0], status='Imported')
if len(searchRes) == 1:
extra.append(
(searchRes[0], [('%.2f' % x).zfill(5) for x in tup[1]]))
else:
total.append((tup[0], [('%.2f' % x).zfill(5) for x in tup[1]]))
print('Create %s ...' % ('AllCredits_%s.txt' % sheetNr),
end='',
flush=True)
writeNameCreditLists(path, 'AllCredits_%s.txt' % sheetNr, total)
print('[OK]')
extTutDict = {}
for elem in extra:
eTutor = elem[0]['ExtTut']
if eTutor in extTutDict:
extTutDict[eTutor].append((elem[0]['Name'], elem[1]))
else:
extTutDict[eTutor] = [(elem[0]['Name'], elem[1])]
for k, v in extTutDict.items():
fname = 'AllCredits_%s_%s_%s.txt' % (sheetNr, fm.getTFirstName(),
k.replace(' ', '-'))
print('Create %s ...' % fname, end='', flush=True)
writeNameCreditLists(path, fname, v)
print('[OK]')
def test_plot_cmb(self):
base = pdirname(__file__)
f_pwr = pjoin(base, 'data/dtu10mw_v1.pwr')
f_cmb = pjoin(base, 'data/campbell_diagram.cmb')
dfp, dff, dfd = read_cmb_all(f_cmb, f_pwr=f_pwr)
cmb = PlotCampbell(dfp['V [m/s]'].values, dff, dfd)
fig, axes = subplots(nrows=2, ncols=1, figsize=(8, 10))
ax = axes[0, 0]
ax = cmb.plot_freq(ax, col='k', mark='^', ls='-', modes='all')
ax = plot_add_ps(ax, dfp['V [m/s]'], dfp['Speed [rpm]'], ps=[1, 3, 6])
ax = axes[1, 0]
ax = cmb.plot_damp(ax, col='k', mark='^', ls='-', modes=10)
def main():
# Read and Save the Original Input File to Work From
with open(input_file_og, 'r') as ipf_og:
for idx, line in enumerate(ipf_og.readlines()):
line_text.append(line.strip())
# Make and Write a Modified Input File
input_file_mod = pjoin(
pdirname(input_file_og),
os.path.splitext(pbasename(input_file_og))[0] + '_cbrain.txt')
#input_file_mod = pjoin(pdirname(input_file_og),pbasename(input_file_og) + '_cbrain.txt')
with open(input_file_mod, 'w+') as ipf_mod:
for idx in range(len(line_text)):
mod_line_text = line_text[idx].replace(
"/CBRAIN_PROCESSING_LOCATION", real_cbrain_path)
ipf_mod.write(mod_line_text + "\n")
def run(self):
if not pexists(self.directory):
makedirs(self.directory)
else:
if isdir(self.directory):
if listdir(self.directory):
error('Directory is not empty: ' + self.directory)
return -1
else:
error('This is not directory: ' + self.directory)
return -1
try:
src_type, src_value = self.probe_source_type()
if self.import_source(src_type, src_value) != 0:
error('Import failed')
if pexists(pjoin(self._directory, Settings.CONTROL_PATH)):
info('{} folder exists!'.format(Settings.CONTROL_PATH))
return 0
# copy template control files
info('Copy default CONTROL files to {}'.format(
pjoin(self.directory, Settings.CONTROL_PATH)))
copytree(pjoin(Settings.TEMPLATE_PATH, Settings.CONTROL_PATH),
pjoin(self.directory, Settings.CONTROL_PATH), True)
info('Modify package name to ' + self._args.project)
# sed control, rules, *.init, *.conf
files_check = ('control', 'rules', 'foobar.init', 'foobar.conf')
for fn in files_check:
fp = pjoin(self.directory, Settings.CONTROL_PATH, fn)
for line in fileinput.input(fp, inplace=True):
print line.replace(Settings.DEFAULT_PACKAGE,
self._args.project),
# mv foobar.* to self._args.project.*
for fn in glob(pjoin(self.directory, Settings.CONTROL_PATH,
Settings.DEFAULT_PACKAGE + '.*')):
dest = pjoin(pdirname(fn),
self._args.project + fn[fn.index('.'):])
move(fn, dest)
except Exception as e:
error(e)
return -1
return 0
def rename_ctrl_files(self):
# rename
if self._args.project != Settings.DEFAULT_PACKAGE:
info('Modify package name to ' + self._args.project)
# sed control, rules, *.init, *.conf
files_check = ('control', 'rules', 'foobar.init', 'foobar.conf')
for fn in files_check:
fp = pjoin(self.directory, Settings.CONTROL_PATH, fn)
for line in fileinput.input(fp, inplace=True):
print line.replace(Settings.DEFAULT_PACKAGE,
self._args.project),
# mv foobar.* to self._args.project.*
for fn in glob(pjoin(self.directory, Settings.CONTROL_PATH,
Settings.DEFAULT_PACKAGE + '.*')):
dest = pjoin(pdirname(fn),
self._args.project + fn[fn.index('.'):])
move(fn, dest)
def rename_ctrl_files(self):
# rename
if self._args.project != Settings.DEFAULT_PACKAGE:
info('Modify package name to ' + self._args.project)
# sed control, rules, *.init, *.conf
files_check = ('control', 'rules', 'foobar.init', 'foobar.conf')
for fn in files_check:
fp = pjoin(self.directory, Settings.CONTROL_PATH, fn)
for line in fileinput.input(fp, inplace=True):
print line.replace(Settings.DEFAULT_PACKAGE,
self._args.project),
# mv foobar.* to self._args.project.*
for fn in glob(
pjoin(self.directory, Settings.CONTROL_PATH,
Settings.DEFAULT_PACKAGE + '.*')):
dest = pjoin(pdirname(fn),
self._args.project + fn[fn.index('.'):])
move(fn, dest)
def _import_archive(self, filename, directory):
if filename.startswith('http://') \
or filename.startswith('https://') \
or filename.startswith('ftp://'):
download_file = self._download(filename)
filename = download_file
if filename is None:
return -1
try:
archive = Archive()
ftype = archive.file_type(filename)
if ftype is None:
error('Invalid archive format: ' + filename)
return -1
archive.decompress(filename, directory, ftype)
finally:
if download_file is not None:
rmtree(pdirname(download_file))
return 0
def sendFeedbackMails(wpath, sendable=False):
fm = FolderManager(wpath)
#print(os.path.basename(os.path.dirname(wpath)))
sheetNr = pbasename(pdirname(wpath)).split('_')[1]
studs = fm.getAllStudents(status='Local;Imported')
mailHtmlDict = prepareMails(studs, fm.getMyEmailData(), wpath,
fm.getReturnTemplatePath())
if sendable:
quest = 'Do u rly want to send Feedbackmails to ur students? (Y|n)'
sendable = input(quest) == 'Y'
else:
print(
'Mails prepared. To send it after prepareing use -S as console parameter'
)
if sendable:
with Sender(acc=fm.getMailAcc()) as sender:
sender.allowed = sendable
sender.sendFeedbackMails(fm.getSubject(), sheetNr, mailHtmlDict)
def F2pyEmitter(target, source, env):
build_dir = pdirname(str(target[0]))
if is_pyf(str(source[0])):
# target is (in this order):
# - the C file which will contain the generated code
# - the fortranobject.c file
# - the f2py fortran wrapper
basename = get_f2py_modulename_from_node(source[0])
ntarget = []
cgename = (CGEN_TEMPLATE % basename) + env['CFILESUFFIX']
ntarget.append(default_fs.Entry(pjoin(build_dir, cgename)))
fobj = pjoin(build_dir, mangle_fortranobject(basename, FOBJECT_FILE))
ntarget.append(default_fs.Entry(fobj))
f2pywrap = pjoin(build_dir, FWRAP_TEMPLATE % basename)
ntarget.append(default_fs.Entry(f2pywrap))
else:
ntarget = target
fobj = pjoin(build_dir,
mangle_fortranobject(str(target[0]), FOBJECT_FILE))
ntarget.append(default_fs.Entry(fobj))
return (ntarget, source)
def F2pyEmitter(target, source, env):
build_dir = pdirname(str(target[0]))
if is_pyf(str(source[0])):
# target is (in this order):
# - the C file which will contain the generated code
# - the fortranobject.c file
# - the f2py fortran wrapper
basename = get_f2py_modulename_from_node(source[0])
ntarget = []
cgename = (CGEN_TEMPLATE % basename) + env['CFILESUFFIX']
ntarget.append(default_fs.Entry(pjoin(build_dir, cgename)))
fobj = pjoin(build_dir, mangle_fortranobject(basename, FOBJECT_FILE))
ntarget.append(default_fs.Entry(fobj))
f2pywrap = pjoin(build_dir, FWRAP_TEMPLATE % basename)
ntarget.append(default_fs.Entry(f2pywrap))
else:
ntarget = target
fobj = pjoin(build_dir, mangle_fortranobject(str(target[0]),
FOBJECT_FILE))
ntarget.append(default_fs.Entry(fobj))
return (ntarget, source)
def main():
# Set Up
checkscript, ext = psplit(pbasename(checkscript_ver))
os.chdir(version_check_location)
summary_file = pjoin(version_check_location,
"Mass_Verification_Summary.txt")
# Main Caller
# Crawl through folders starting at base searching for matlab and octave results to verify
matlabfile = ""
octavefile = ""
for option in os.listdir(version_check_location):
if os.path.isdir(pjoin(version_check_location, option)):
for environment in os.listdir(pjoin(version_check_location,
option)):
input_txt_file_path = pjoin(version_check_location, option,
environment)
if os.path.isdir(input_txt_file_path):
print("Checking {} for OPPNI input files".format(
pjoin(version_check_location, option, environment)))
# for processed in os.listdir(pjoin(version_check_location, option, environment)):
# if os.path.isdir(pjoin(version_check_location,option,environment,processed)):
# input_txt_file_path = pjoin(version_check_location, option, environment, processed)
# Gets input files ready
if "matlab" in environment:
matlabfile = pjoin(input_txt_file_path,
matlab_input_file_name)
if "octave" in environment:
octavefile = pjoin(input_txt_file_path,
octave_input_file_name)
# pass
pass
pass
if matlabfile == "":
print("Unable to locate OPPNI Matlab input.txt: ",
input_txt_file_path)
continue
if octavefile == "":
print("Unable to locate OPPNI Octave input.txt: ",
input_txt_file_path)
continue
pass
if matlabfile == "" or octavefile == "":
print(
"Unable to locate file to varify, moving to next directory path"
)
continue
# Get call ready
# check for successful OPPNI completion (matlab)
out_dir = os.path.dirname(os.path.abspath(matlabfile))
fileList = glob.glob(
pjoin(out_dir, "processing_input_file_*", "*.pkl"))
out_dir = os.path.dirname(os.path.abspath(fileList[0]))
print("Options file is: {}".format(fileList[0]))
prev_proc_status, prev_options, prev_input_file_all, failed_sub_file, failed_spnorm_file, all_subjects = update_proc_status(
out_dir)
if not prev_proc_status.all_done:
print(
'Previous processing is incomplete for:{} \n Moving on to next result set.'
.format(matlabfile))
continue
else:
print("matlabfile = ", matlabfile)
#check for successful OPPNI completion (octave)
out_dir = os.path.dirname(os.path.abspath(octavefile))
fileList = glob.glob(
pjoin(out_dir, "processing_input_file_*", "*.pkl"))
out_dir = os.path.dirname(os.path.abspath(fileList[0]))
print("Options file is: {}".format(fileList[0]))
prev_proc_status, prev_options, prev_input_file_all, failed_sub_file, failed_spnorm_file, all_subjects = update_proc_status(
out_dir)
if not prev_proc_status.all_done:
print(
'Previous processing is incomplete for:{} \n Moving on to next result set.'
.format(octavefile))
continue
else:
print("octavefile = ", octavefile)
savefile = pjoin(version_check_location, option,
"MvsO_results.txt")
print("Results will be saved to: ", savefile)
call_command = "addpath(genpath('" + matlab_path + "')); " + checkscript + "('" + matlabfile + "','" + octavefile + "','" + savefile + "'); exit;"
print("validation command line: ", call_command)
#Call matlab from command terminal to run Nathans Version Checker
subprocess.call(
["matlab", "-nodesktop", "-nosplash", "-r", call_command])
# Load file and check for all [OK]
totalok = 0
with open(savefile, 'r') as result:
for idx, line in enumerate(result.readlines()):
if "[OK]" in line:
totalok += 1
summary_string = "Verified " + str(
totalok) + "/14 [OK] for: " + pbasename(pdirname(savefile))
# Save results into summary
with open(summary_file, 'w+') as summary:
summary.write(summary_string)
def _call_scons(self, scons_exec, sconscript, pkg_name, pkg_path,
bootstrapping):
# XXX: when a scons script is missing, scons only prints warnings, and
# does not return a failure (status is 0). We have to detect this from
# distutils (this cannot work for recursive scons builds...)
# XXX: passing everything at command line may cause some trouble where
# there is a size limitation ? What is the standard solution in thise
# case ?
cmd = [scons_exec, "-f", sconscript, '-I.']
if self.jobs:
cmd.append(" --jobs=%d" % int(self.jobs))
if self.inplace:
cmd.append("inplace=1")
cmd.append('scons_tool_path="%s"' % self.scons_tool_path)
cmd.append('src_dir="%s"' % pdirname(sconscript))
cmd.append('pkg_path="%s"' % pkg_path)
cmd.append('pkg_name="%s"' % pkg_name)
cmd.append('log_level=%s' % self.log_level)
#cmd.append('distutils_libdir=%s' % protect_path(pjoin(self.build_lib,
# pdirname(sconscript))))
cmd.append('distutils_libdir=%s' %
protect_path(get_distutils_libdir(self, pkg_name)))
cmd.append('distutils_clibdir=%s' %
protect_path(get_distutils_clibdir(self, pkg_name)))
prefix = get_distutils_install_prefix(pkg_name, self.inplace)
cmd.append('distutils_install_prefix=%s' % protect_path(prefix))
if not self._bypass_distutils_cc:
cmd.append('cc_opt=%s' % self.scons_compiler)
if self.scons_compiler_path:
cmd.append('cc_opt_path=%s' % self.scons_compiler_path)
else:
cmd.append('cc_opt=%s' % self.scons_compiler)
cmd.append('debug=%s' % self.debug)
if self.scons_fcompiler:
cmd.append('f77_opt=%s' % self.scons_fcompiler)
if self.scons_fcompiler_path:
cmd.append('f77_opt_path=%s' % self.scons_fcompiler_path)
if self.scons_cxxcompiler:
cmd.append('cxx_opt=%s' % self.scons_cxxcompiler)
if self.scons_cxxcompiler_path:
cmd.append('cxx_opt_path=%s' % self.scons_cxxcompiler_path)
cmd.append('include_bootstrap=%s' %
dirl_to_str(get_numpy_include_dirs(sconscript)))
cmd.append('bypass=%s' % self.bypass)
cmd.append('import_env=%s' % self.import_env)
if self.silent:
if int(self.silent) == 2:
cmd.append('-Q')
elif int(self.silent) == 3:
cmd.append('-s')
cmd.append('silent=%d' % int(self.silent))
cmd.append('bootstrapping=%d' % bootstrapping)
cmdstr = ' '.join(cmd)
if int(self.silent) < 1:
log.info("Executing scons command (pkg is %s): %s ", pkg_name,
cmdstr)
else:
log.info("======== Executing scons command for pkg %s =========",
pkg_name)
st = os.system(cmdstr)
if st:
#print "status is %d" % st
msg = "Error while executing scons command."
msg += " See above for more information.\n"
msg += """\
If you think it is a problem in numscons, you can also try executing the scons
command with --log-level option for more detailed output of what numscons is
doing, for example --log-level=0; the lowest the level is, the more detailed
the output it."""
raise DistutilsExecError(msg)
#!/usr/bin/env python
import sys
from os import getenv
from os.path import (join as pjoin,
dirname as pdirname,
abspath as pabspath,
)
if sys.argv[0].startswith('/usr'):
PREFIX = '/usr/share/qdk2'
QDK_BINARY = 'QDK'
else:
PREFIX = pdirname(pdirname(pabspath(sys.argv[0])))
QDK_BINARY = 'QDK_2.2'
VERSION = 'v0.10-1-g04d7ce1'
class Settings(object):
DEBUG = False if getenv('DEBUG') is None else True
QPKG_VERSION = '2.2'
CONTROL_PATH = 'QNAP'
SUPPORT_TEMPLATES = ('c-cpp')
DEFAULT_PROJECT = 'new_project'
DEFAULT_PACKAGE = 'foobar'
SAMPLES_PATH = pjoin(PREFIX, 'samples')
TEMPLATE_PATH = pjoin(PREFIX, 'template')
TEMPLATE_V1_PATH = pjoin(PREFIX, QDK_BINARY, 'template')
#!/usr/bin/env python
import sys
from os import getenv
from os.path import (
join as pjoin,
dirname as pdirname,
abspath as pabspath,
)
if sys.argv[0].startswith('/usr'):
PREFIX = '/usr/share/qdk2'
QDK_BINARY = 'QDK'
else:
PREFIX = pdirname(pdirname(pabspath(sys.argv[0])))
QDK_BINARY = 'QDK_2.x'
VERSION = 'v0.10-1-g04d7ce1'
class Settings(object):
DEBUG = False if getenv('DEBUG') is None else True
QPKG_VERSION = '2.2'
CONTROL_PATH = 'QNAP'
SUPPORT_TEMPLATES = ('c-cpp')
DEFAULT_PROJECT = 'new_project'
DEFAULT_PACKAGE = 'foobar'
SAMPLES_PATH = pjoin(PREFIX, 'samples')
TEMPLATE_PATH = pjoin(PREFIX, 'template')
TEMPLATE_V1_PATH = pjoin(PREFIX, QDK_BINARY, 'template')
QBUILD = pjoin(PREFIX, QDK_BINARY, 'bin', 'qbuild')
"""Misc utilities which did not find they way elsewhere."""
import sys
import os
import re
from os.path import join as pjoin, dirname as pdirname, basename as pbasename
import numscons
_CONFDIR = pjoin(pdirname(numscons.__file__), 'configurations')
# XXX: this is ugly, better find the mathlibs with a checker
# XXX: this had nothing to do here, too...
def scons_get_paths(paths):
return paths.split(os.pathsep)
def get_scons_path():
"""Returns the name of the directory where our local scons is."""
return pjoin(pdirname(numscons.__file__), 'scons-local')
def get_scons_build_dir():
"""Return the top path where everything produced by scons will be put.
The path is relative to the top setup.py"""
return pjoin('build', 'scons')
def get_scons_configres_dir():
"""Return the top path where everything produced by scons will be put.
The path is relative to the top setup.py"""
def check_api_version(apiversion):
return _check_api_version(apiversion, pdirname(code_generators.__file__))
def get_scons_path():
"""Returns the name of the directory where our local scons is."""
return pjoin(pdirname(numscons.__file__), 'scons-local')
def get_local_toolpaths():
"""Returns the full pathname for the numscons tools directory."""
return [pdirname(numscons.tools.__file__)]
def generate_from_template_emitter(target, source, env):
base, ext = split_ext(pbasename(str(source[0])))
t = pjoin(pdirname(str(target[0])), base)
return ([t], source)
bootstrap = 0
for sconscript, pre_hook, post_hook, pkg_name in zip(sconscripts,
pre_hooks, post_hooks,
pkg_names):
if pre_hook:
pre_hook()
if sconscript:
cmd = [scons_exec, "-f", sconscript, '-I.']
if self.jobs:
cmd.append(" --jobs=%d" % int(self.jobs))
if self.inplace:
cmd.append("inplace=1")
cmd.append('scons_tool_path="%s"' % self.scons_tool_path)
cmd.append('src_dir="%s"' % pdirname(sconscript))
cmd.append('pkg_name="%s"' % pkg_name)
cmd.append('log_level=%s' % self.log_level)
#cmd.append('distutils_libdir=%s' % protect_path(pjoin(self.build_lib,
# pdirname(sconscript))))
cmd.append('distutils_libdir=%s' %
protect_path(get_distutils_libdir(self, pkg_name)))
if not self._bypass_distutils_cc:
cmd.append('cc_opt=%s' % self.scons_compiler)
cmd.append('cc_opt_path=%s' % self.scons_compiler_path)
else:
cmd.append('cc_opt=%s' % self.scons_compiler)
if self.fcompiler:
def setUp(self):
self.fbase = pdirname(__file__)
self.fpath_linear = pjoin(self.fbase,
'data/controller_input_linear.txt')
self.fpath_quad = pjoin(self.fbase,
'data/controller_input_quadratic.txt')
def check_api_version(apiversion):
return _check_api_version(apiversion, pdirname(code_generators.__file__))
else:
bootstrap = 0
for sconscript, pre_hook, post_hook, pkg_name in zip(
sconscripts, pre_hooks, post_hooks, pkg_names):
if pre_hook:
pre_hook()
if sconscript:
cmd = [scons_exec, "-f", sconscript, '-I.']
if self.jobs:
cmd.append(" --jobs=%d" % int(self.jobs))
if self.inplace:
cmd.append("inplace=1")
cmd.append('scons_tool_path="%s"' % self.scons_tool_path)
cmd.append('src_dir="%s"' % pdirname(sconscript))
cmd.append('pkg_name="%s"' % pkg_name)
cmd.append('log_level=%s' % self.log_level)
#cmd.append('distutils_libdir=%s' % protect_path(pjoin(self.build_lib,
# pdirname(sconscript))))
cmd.append('distutils_libdir=%s' %
protect_path(get_distutils_libdir(self, pkg_name)))
if not self._bypass_distutils_cc:
cmd.append('cc_opt=%s' % self.scons_compiler)
cmd.append('cc_opt_path=%s' % self.scons_compiler_path)
else:
cmd.append('cc_opt=%s' % self.scons_compiler)
if self.fcompiler:
cmd.append('f77_opt=%s' % dist2sconsfc(self.fcompiler))
def get_local_toolpaths():
"""Returns the full pathname for the numscons tools directory."""
return [pdirname(numscons.tools.__file__)]
def generate_from_template_emitter(target, source, env):
base, ext = split_ext(pbasename(str(source[0])))
t = pjoin(pdirname(str(target[0])), base)
return ([t], source)
def generate_from_template_emitter(target, source, env):
"""Scons emitter for both C and Fortran generated files from template (.src
files)."""
base = splitext(pbasename(str(source[0])))[0]
t = pjoin(pdirname(str(target[0])), base)
return ([t], source)
def get_scons_path():
"""Returns the name of the directory where our local scons is."""
return pjoin(pdirname(numscons.__file__), 'scons-local')
def generate_from_template_emitter(target, source, env):
"""Scons emitter for both C and Fortran generated files from template (.src
files)."""
base = splitext(pbasename(str(source[0])))[0]
t = pjoin(pdirname(str(target[0])), base)
return ([t], source)
def _call_scons(self, scons_exec, sconscript, pkg_name, bootstrapping):
# XXX: when a scons script is missing, scons only prints warnings, and
# does not return a failure (status is 0). We have to detect this from
# distutils (this cannot work for recursive scons builds...)
# XXX: passing everything at command line may cause some trouble where
# there is a size limitation ? What is the standard solution in thise
# case ?
cmd = [scons_exec, "-f", sconscript, "-I."]
if self.jobs:
cmd.append(" --jobs=%d" % int(self.jobs))
if self.inplace:
cmd.append("inplace=1")
cmd.append('scons_tool_path="%s"' % self.scons_tool_path)
cmd.append('src_dir="%s"' % pdirname(sconscript))
cmd.append('pkg_name="%s"' % pkg_name)
cmd.append("log_level=%s" % self.log_level)
# cmd.append('distutils_libdir=%s' % protect_path(pjoin(self.build_lib,
# pdirname(sconscript))))
cmd.append("distutils_libdir=%s" % protect_path(get_distutils_libdir(self, pkg_name)))
cmd.append("distutils_clibdir=%s" % protect_path(get_distutils_clibdir(self, pkg_name)))
prefix = get_distutils_install_prefix(pkg_name, self.inplace)
cmd.append("distutils_install_prefix=%s" % protect_path(prefix))
if not self._bypass_distutils_cc:
cmd.append("cc_opt=%s" % self.scons_compiler)
if self.scons_compiler_path:
cmd.append("cc_opt_path=%s" % self.scons_compiler_path)
else:
cmd.append("cc_opt=%s" % self.scons_compiler)
if self.scons_fcompiler:
cmd.append("f77_opt=%s" % self.scons_fcompiler)
if self.scons_fcompiler_path:
cmd.append("f77_opt_path=%s" % self.scons_fcompiler_path)
if self.scons_cxxcompiler:
cmd.append("cxx_opt=%s" % self.scons_cxxcompiler)
if self.scons_cxxcompiler_path:
cmd.append("cxx_opt_path=%s" % self.scons_cxxcompiler_path)
cmd.append("include_bootstrap=%s" % dirl_to_str(get_numpy_include_dirs(sconscript)))
cmd.append("bypass=%s" % self.bypass)
cmd.append("import_env=%s" % self.import_env)
if self.silent:
if int(self.silent) == 2:
cmd.append("-Q")
elif int(self.silent) == 3:
cmd.append("-s")
cmd.append("silent=%d" % int(self.silent))
cmd.append("bootstrapping=%d" % bootstrapping)
cmdstr = " ".join(cmd)
if int(self.silent) < 1:
log.info("Executing scons command (pkg is %s): %s ", pkg_name, cmdstr)
else:
log.info("======== Executing scons command for pkg %s =========", pkg_name)
st = os.system(cmdstr)
if st:
# print "status is %d" % st
msg = "Error while executing scons command."
msg += " See above for more information.\n"
msg += """\
If you think it is a problem in numscons, you can also try executing the scons
command with --log-level option for more detailed output of what numscons is
doing, for example --log-level=0; the lowest the level is, the more detailed
the output it."""
raise DistutilsExecError(msg)
def main(oppni_ver, input_file_og, pipeline_file, reference_file):
# Read and Save the Original Input File to Work From
with open(input_file_og, 'r') as ipf_og:
for idx, line in enumerate(ipf_og.readlines()):
line_text.append(line.strip())
out_text.append(get_out_dir_line(line))
base_out_dir = pdirname(out_text[0])
# Main Loop : Apply Conditions
for model in CODES_ANALYSIS_MODELS:
for pattern in SLICE_TIMING_PATTERNS:
for dflag in DEOBLIQUE:
# Generate Suffix
suffix = "Opt_" + str(model) + '_' + str(pattern) + '_' + str(
dflag)
# 2 Execution Methods
for env in EXE_METHODS:
# Create New Folders
base_cond_dir = pjoin(base_out_dir, suffix)
base_cond_env_dir = pjoin(base_cond_dir, str(env))
os.makedirs(base_cond_dir, exist_ok=True)
os.makedirs(base_cond_env_dir, exist_ok=True)
# Make and Write a Modified Input File
input_file_mod = pjoin(base_cond_env_dir, 'input_file.txt')
with open(input_file_mod, 'w+') as ipf_mod:
for idx in range(len(line_text)):
# prepending it with OUT= to restrict the sub to only OUT, and not elsewhere such as TASK=
mod_out_text = 'OUT=' + pjoin(
pdirname(out_text[idx]), suffix, str(env),
pbasename(out_text[idx]))
# rep_text = 'OUT={}'.format(base_out_dir)
rep_text = 'OUT={}'.format(
pjoin(base_out_dir, pbasename(out_text[idx])))
mod_line_text = line_text[idx].replace(
rep_text, mod_out_text)
#out_text[idx] = 'OUT={}'.format(base_out_dir)
#mod_line_text = line_text[idx].replace(out_text[idx], mod_out_text)
print(
"rep_text = [{}]\nmod_out_text = [{}]\nmod_line_text = [{}]\n"
.format(rep_text, mod_out_text, mod_line_text))
#DEBUG
#if input("Continue? (Y)").upper() != "Y":
# sys.exit()
ipf_mod.write(mod_line_text + "\n")
# Make and Write a Modified Command File
commandfile = pjoin(
pjoin(base_cond_env_dir,
"cmd_" + suffix + "_" + str(env) + '.sh'))
with open(commandfile, "w+") as cmd_fid:
submit_string = (
'python ' + oppni_ver + ' -i ' + input_file_mod +
' -c ' + pipeline_file + ' -a ' + str(model) +
' -r ' + reference_file + ' ' + volume +
' --TPATTERN ' + str(pattern) + ' --convolve ' +
convolve + ' -m ' + metric + ' -e ' + str(env) +
' ' + str(dflag) + ' ' + cluster + ' --contrast ' +
contrast + ' ' + vasc_mask + ' ' + blur + ' ' +
control_motion_artifact + ' ' + control_wm_bias +
' ' + output_nii_also + ' ' + dry_run + ' ' +
run_locally)
cmd_fid.write(submit_string)
# Run OPPNI in Terminal
# Make Sure that System Settings are Correct
exit_code = os.system(
submit_string) # TURN THIS ON AND OFF
if exit_code != 0:
print("\nExit Code {} was returned for command [{}]".
format(exit_code, submit_string))
if (input("\nContinue with next command? (Y/N):").
upper() == 'N'):
sys.exit()
