def __init__(self, **kw):
Environment.__init__(*(self, ), **kw)
# self.EnsureSConsVersion(0,96)
opts = {
'TIMER': 'Whether to time execution',
'CHECKPAR': 'Whether to check parameters',
'ENVIRON': 'Additional environment settings',
'CLUSTER': 'Nodes available on a cluster',
'BATCH': 'Parameter file for batch jobs on a cluster'
}
rsf.conf.set_options(self, opts)
root = self.get('RSFROOT', rsf.prog.RSFROOT)
self.bindir = os.path.join(root, 'bin')
self.sfpen = os.path.join(self.bindir, 'sfpen')
self.pspen = os.path.join(self.bindir, 'pspen')
self.vppen = os.path.join(self.bindir, 'vppen')
# self.runonnode = os.path.join(self.bindir,'runonnode')
self.figs = os.environ.get(
'RSFFIGS', os.path.join(root, 'share', 'madagascar', 'figs'))
cwd = os.getcwd()
self.cwd = cwd
# path for binary files
self.path = rsf.path.getpath(cwd)
tmpdatapath = os.environ.get('TMPDATAPATH', self.path)
rsf.path.sconsign(self)
self.resdir = resdir
self.figdir = re.sub('.*\/((?:[^\/]+)\/(?:[^\/]+)\/(?:[^\/]+))$',
self.figs + '/\\1', cwd)
self.progsuffix = self['PROGSUFFIX']
# Keep certain environmental variables in the environment
for env in keepenv:
getenv = os.environ.get(env)
if getenv:
self.Append(ENV={env: getenv})
self.hostname = socket.gethostname()
# Keep environmental variables needed for SLURM
for env in list(os.environ.keys()):
if 'SLURM_' == env[:
6] or 'TACC_' == env[:
5] or '_ModuleTable' == env[:
12]:
self.Append(ENV={env: os.environ[env]})
if sys.platform[:6] == 'cygwin':
exe = ''
else:
exe = '.exe'
libdir = os.path.join(root, 'lib')
incdir = os.path.join(root, 'include')
self.Append(ENV={
'RSFROOT':
root,
'DATAPATH':
self.path,
'OMP_NUM_THREADS':
os.environ.get('OMP_NUM_THREADS', rsf.node.cpus()),
'TMPDATAPATH':
tmpdatapath,
'XAUTHORITY':
os.environ.get('XAUTHORITY',
os.path.join(os.environ.get('HOME'), '.Xauthority'))
},
BUILDERS={
'Retrieve': Retrieve,
'Test': Test,
'Echo': Echo
},
LIBPATH=[libdir],
CPPPATH=[incdir],
F90PATH=[incdir],
LIBS=[libs],
PROGSUFFIX=exe)
self.Prepend(LIBS=[self.get('DYNLIB', '') + 'rsf'])
minesjtk = self.get('MINESJTK', None)
usejava = self.get('JAVA_HOME', None)
if not usejava: usejava = self.get('JAVA_SDK', None)
if minesjtk or usejava:
classpath = []
classpath.append(os.path.join(libdir, 'rsf.jar'))
userclasspath = os.environ.get('CLASSPATH', None)
if userclasspath: classpath.append(userclasspath)
if minesjtk:
classpath.append(minesjtk)
self.Append(JAVACLASSPATH=':'.join(classpath))
classpath.append('.')
self.Append(ENV={'CLASSPATH': ':'.join(classpath)})
path = {
'darwin': '/opt/local/bin',
'irix': '/usr/freeware/bin',
'cygwin': '/usr/X11R6/bin:/usr/lib/lapack:' + libdir
}
for plat in list(path.keys()):
if sys.platform[:len(plat)] == plat:
self['ENV']['PATH'] = ':'.join(
[path[plat], self['ENV']['PATH']])
pythonpath = os.path.join(sys.prefix, 'bin')
if os.path.isdir(pythonpath):
self['ENV']['PATH'] = ':'.join([pythonpath, self['ENV']['PATH']])
if sys.platform[:6] == 'cygwin':
self['ENV']['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
self['PROGPREFIX'] = ''
self.view = []
self.prnt = []
self.lock = []
self.test = []
self.coms = []
self.data = []
self.rest = []
sys.path.append('../../../Recipes')
timer = self.get('TIMER')
if timer and timer[0] != 'n' and timer[0] != '0':
self.timer = WhereIs('time') + ' '
else:
self.timer = ''
self.mpirun = self.get('MPIRUN', WhereIs('ibrun') or WhereIs('mpirun'))
checkpar = self.get('CHECKPAR')
self.checkpar = checkpar and checkpar[0] != 'n' and checkpar[0] != '0'
self.environ = self.get('ENVIRON', '')
self.batch = self.get('BATCH')
self.jobs = GetOption('num_jobs') # getting information from scons -j
cluster = self.get('CLUSTER',
os.environ.get('RSF_CLUSTER', 'localhost 1'))
hosts = cluster.split()
self.nodes = []
for i in range(1, len(hosts), 2):
nh = int(hosts[i])
self.nodes.extend([hosts[i - 1]] * nh)
self.ip = 0
# self.nodes is a list of CPUs
# self.jobs is the number of jobs
# self.ip is the current CPU
for key in list(self['ENV'].keys()):
self.environ = self.environ + " %s='%s'" % (key, self['ENV'][key])
def __init__(self, **kw):
kw.update({'tools': []})
apply(Environment.__init__, (self, ), kw)
rsf.conf.set_options(self)
sourceforge = 'http://sourceforge.net/p/rsf/code/HEAD/tree/trunk'
self.Append(ENV={
'XAUTHORITY':
os.path.join(os.environ.get('HOME'), '.Xauthority'),
'DISPLAY':
os.environ.get('DISPLAY'),
'RSF_REPOSITORY':
os.environ.get('RSF_REPOSITORY', sourceforge),
'RSF_ENSCRIPT':
WhereIs('enscript'),
'HOME':
os.environ.get('HOME')
},
SCANNERS=LaTeXS,
BUILDERS={
'Latify': Latify,
'Pdf': Pdf,
'Wiki': Wiki,
'Build': Build,
'Color': Color,
'Figs': Figs
})
path = {
'darwin': ['/sw/bin', '/opt/local/bin'],
'irix': ['/usr/freeware/bin']
}
for plat in path.keys():
if sys.platform[:len(plat)] == plat:
for pathdir in filter(os.path.isdir, path[plat]):
self['ENV']['PATH'] = ':'.join(
[pathdir, self['ENV']['PATH']])
tree = rsf.path.dirtree()
root = self.get('RSFROOT', rsf.prog.RSFROOT)
self.docdir = os.environ.get('RSFBOOK',
os.path.join(root, 'share', 'madagascar'))
self.figdir = os.environ.get('RSFFIGS',
os.path.join(self.docdir, 'figs'))
for level in tree:
if level:
self.docdir = os.path.join(self.docdir, level)
rsf.path.mkdir(self.docdir)
datapath = rsf.path.datapath()
self.path = os.path.dirname(datapath)
if datapath[:2] != './':
for level in tree[1:]:
self.path = os.path.join(self.path, level)
rsf.path.mkdir(self.path)
self.path = os.path.join(self.path, os.path.basename(datapath))
rsf.path.sconsign(self)
if pdfread:
self.Append(BUILDERS={'Read': Read, 'Print': Print})
if epstopdf:
self.Append(BUILDERS={'PDFBuild': PDFBuild})
if fig2dev:
self.Append(BUILDERS={'XFig': XFig})
if latex2html:
self.Append(BUILDERS={'HTML': HTML, 'PNGBuild': PNGBuild})
self.imgs = []
if (acroread and ps2eps) or pdf2ps:
self.Append(BUILDERS={'PSBuild': PSBuild})
if epstopdf:
if mathematica:
self.Append(BUILDERS={'Math': Math})
if gnuplot:
self.Append(BUILDERS={'Gnuplot': Gnuplot})
if matlab:
self.Append(BUILDERS={'Matlab': Matlab})
if haspylab:
self.Append(BUILDERS={'Pylab': Pylab})
if sage:
self.Append(BUILDERS={'Sage': Sage})
self.scons = []
self.figs = []
self.Dir()
def __init__(self,**kw):
apply(Environment.__init__,(self,),kw)
self.EnsureSConsVersion(0,96)
opts = {
'TIMER':'Whether to time execution',
'CHECKPAR':'Whether to check parameters',
'ENVIRON':'Additional environment settings',
'CLUSTER':'Nodes available on a cluster',
'MPIRUN':'mpirun command'
}
rsf.conf.set_options(self,opts)
root = self.get('RSFROOT',rsf.prog.RSFROOT)
self.bindir = os.path.join(root,'bin')
self.sfpen = os.path.join(self.bindir,'sfpen')
self.pspen = os.path.join(self.bindir,'pspen')
self.vppen = os.path.join(self.bindir,'vppen')
self.figs = os.environ.get('RSFFIGS',os.path.join(root,'share','madagascar','figs'))
cwd = os.getcwd()
self.cwd = cwd
# path for binary files
self.path = rsf.path.getpath(cwd)
tmpdatapath = os.environ.get('TMPDATAPATH',self.path)
rsf.path.sconsign(self)
self.resdir = resdir
self.figdir = re.sub('.*\/((?:[^\/]+)\/(?:[^\/]+)\/(?:[^\/]+))$',
self.figs+'/\\1',cwd)
self.progsuffix = self['PROGSUFFIX']
for env in keepenv:
getenv = os.environ.get(env)
if getenv:
self.Append(ENV={env:getenv})
if sys.platform[:6] == 'cygwin':
exe = ''
else:
exe = '.exe'
libdir = os.path.join(root,'lib')
incdir = os.path.join(root,'include')
self.Append(ENV={'RSFROOT':root,
'DATAPATH':self.path,
'OMP_NUM_THREADS': os.environ.get('OMP_NUM_THREADS',rsf.path.cpus()),
'TMPDATAPATH': tmpdatapath,
'SFPENOPTS': os.environ.get('SFPENOPTS',''),
'XAUTHORITY':
os.environ.get('XAUTHORITY',
os.path.join(os.environ.get('HOME'),
'.Xauthority'))},
BUILDERS={'Retrieve':Retrieve,
'Test':Test,
'Echo':Echo},
LIBPATH=[libdir],
CPPPATH=[incdir],
LIBS=[libs],
PROGSUFFIX=exe)
self.Prepend(LIBS=[self.get('DYNLIB','')+'rsf'])
minesjtk = self.get('MINESJTK',None)
usejava = self.get('JAVA_HOME',None)
if not usejava: usejava = self.get('JAVA_SDK',None)
if minesjtk or usejava:
classpath = []
classpath.append(os.path.join(libdir,'rsf.jar'))
userclasspath = os.environ.get('CLASSPATH',None)
if userclasspath: classpath.append(userclasspath)
if minesjtk:
classpath.append(minesjtk)
self.Append(JAVACLASSPATH=':'.join(classpath))
classpath.append('.')
self.Append(ENV={'CLASSPATH':':'.join(classpath)})
path = {'darwin': '/opt/local/bin',
'irix': '/usr/freeware/bin',
'cygwin': '/usr/X11R6/bin:/usr/lib/lapack'}
for plat in path.keys():
if sys.platform[:len(plat)] == plat:
self['ENV']['PATH'] = ':'.join([path[plat],
self['ENV']['PATH']])
pythonpath = os.path.join(sys.prefix,'bin')
if os.path.isdir(pythonpath):
self['ENV']['PATH'] = ':'.join([pythonpath,
self['ENV']['PATH']])
if sys.platform[:6] == 'cygwin':
self['ENV']['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
self['PROGPREFIX']=''
self.view = []
self.prnt = []
self.lock = []
self.test = []
self.coms = []
self.data = []
self.rest = []
sys.path.append('../../../Recipes')
timer = self.get('TIMER')
if timer and timer[0] != 'n' and timer[0] != '0':
self.timer = WhereIs('time') + ' '
else:
self.timer = ''
self.mpirun = self.get('MPIRUN',WhereIs('ibrun') or WhereIs('mpirun'))
wsplit = os.environ.get('RSF_WSPLIT', None)
if wsplit:
self.wsplit = True
else:
self.wsplit = False
raddenv = os.environ.get('RSF_RADDENV', None)
if raddenv:
self.raddenv = raddenv
else:
self.raddenv = ''
checkpar = self.get('CHECKPAR')
self.checkpar = checkpar and checkpar[0] != 'n' and checkpar[0] != '0'
self.environ = self.get('ENVIRON','')
self.jobs = GetOption('num_jobs')
cluster = self.get('CLUSTER',os.environ.get('RSF_CLUSTER','localhost 1'))
hosts = string.split(cluster)
self.nodes = []
for i in range(1,len(hosts),2):
nh = int(hosts[i])
self.nodes.extend([hosts[i-1]]*nh)
self.taskonnode=[]
for i in range(len(self.nodes)):
self.taskonnode.extend([False])
self.ip = 0
# self.nodes is a list of CPUs
# self.jobs is the number of jobs
# self.ip is the current CPU
for key in self['ENV'].keys():
self.environ = self.environ + ' %s=%s' % (key,self['ENV'][key])
def __init__(self,**kw):
apply(Environment.__init__,(self,),kw)
# self.EnsureSConsVersion(0,96)
opts = {
'TIMER':'Whether to time execution',
'CHECKPAR':'Whether to check parameters',
'ENVIRON':'Additional environment settings',
'CLUSTER':'Nodes available on a cluster',
'BATCH' : 'Parameter file for batch jobs on a cluster'
}
rsf.conf.set_options(self,opts)
root = self.get('RSFROOT',rsf.prog.RSFROOT)
self.bindir = os.path.join(root,'bin')
self.sfpen = os.path.join(self.bindir,'sfpen')
self.pspen = os.path.join(self.bindir,'pspen')
self.vppen = os.path.join(self.bindir,'vppen')
self.runonnode = os.path.join(self.bindir,'runonnode')
self.figs = os.environ.get('RSFFIGS',os.path.join(root,'share','madagascar','figs'))
cwd = os.getcwd()
self.cwd = cwd
# path for binary files
self.path = rsf.path.getpath(cwd)
tmpdatapath = os.environ.get('TMPDATAPATH',self.path)
rsf.path.sconsign(self)
self.resdir = resdir
self.figdir = re.sub('.*\/((?:[^\/]+)\/(?:[^\/]+)\/(?:[^\/]+))$',
self.figs+'/\\1',cwd)
self.progsuffix = self['PROGSUFFIX']
# Keep certain environmental variables in the environment
for env in keepenv:
getenv = os.environ.get(env)
if getenv:
self.Append(ENV={env:getenv})
self.hostname = socket.gethostname()
# Keep environmental variables needed for SLURM
for env in os.environ.keys():
if 'SLURM_' == env[:6] or 'TACC_' == env[:5] or '_ModuleTable' == env[:12]:
self.Append(ENV={env:os.environ[env]})
if sys.platform[:6] == 'cygwin':
exe = ''
else:
exe = '.exe'
libdir = os.path.join(root,'lib')
incdir = os.path.join(root,'include')
self.Append(ENV={'RSFROOT':root,
'DATAPATH':self.path,
'OMP_NUM_THREADS': os.environ.get('OMP_NUM_THREADS',rsf.node.cpus()),
'TMPDATAPATH': tmpdatapath,
'XAUTHORITY':
os.environ.get('XAUTHORITY',
os.path.join(os.environ.get('HOME'),
'.Xauthority'))},
BUILDERS={'Retrieve':Retrieve,
'Test':Test,
'Echo':Echo},
LIBPATH=[libdir],
CPPPATH=[incdir],
F90PATH=[incdir],
LIBS=[libs],
PROGSUFFIX=exe)
self.Prepend(LIBS=[self.get('DYNLIB','')+'rsf'])
minesjtk = self.get('MINESJTK',None)
usejava = self.get('JAVA_HOME',None)
if not usejava: usejava = self.get('JAVA_SDK',None)
if minesjtk or usejava:
classpath = []
classpath.append(os.path.join(libdir,'rsf.jar'))
userclasspath = os.environ.get('CLASSPATH',None)
if userclasspath: classpath.append(userclasspath)
if minesjtk:
classpath.append(minesjtk)
self.Append(JAVACLASSPATH=':'.join(classpath))
classpath.append('.')
self.Append(ENV={'CLASSPATH':':'.join(classpath)})
path = {'darwin': '/opt/local/bin',
'irix': '/usr/freeware/bin',
'cygwin': '/usr/X11R6/bin:/usr/lib/lapack'}
for plat in path.keys():
if sys.platform[:len(plat)] == plat:
self['ENV']['PATH'] = ':'.join([path[plat],
self['ENV']['PATH']])
pythonpath = os.path.join(sys.prefix,'bin')
if os.path.isdir(pythonpath):
self['ENV']['PATH'] = ':'.join([pythonpath,
self['ENV']['PATH']])
if sys.platform[:6] == 'cygwin':
self['ENV']['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
self['PROGPREFIX']=''
self.view = []
self.prnt = []
self.lock = []
self.test = []
self.coms = []
self.data = []
self.rest = []
sys.path.append('../../../Recipes')
timer = self.get('TIMER')
if timer and timer[0] != 'n' and timer[0] != '0':
self.timer = WhereIs('time') + ' '
else:
self.timer = ''
self.mpirun = self.get('MPIRUN',WhereIs('ibrun') or WhereIs('mpirun'))
raddenv = os.environ.get('RSF_RADDENV', None)
if raddenv:
self.raddenv = raddenv
else:
self.raddenv = ''
checkpar = self.get('CHECKPAR')
self.checkpar = checkpar and checkpar[0] != 'n' and checkpar[0] != '0'
self.environ = self.get('ENVIRON','')
self.batch = self.get('BATCH')
self.jobs = GetOption('num_jobs') # getting information from scons -j
cluster = self.get('CLUSTER',os.environ.get('RSF_CLUSTER','localhost 1'))
hosts = string.split(cluster)
self.nodes = []
for i in range(1,len(hosts),2):
nh = int(hosts[i])
self.nodes.extend([hosts[i-1]]*nh)
self.ip = 0
# self.nodes is a list of CPUs
# self.jobs is the number of jobs
# self.ip is the current CPU
# Karl Schleicher kls. I'll move hosts.txt back to the current working
# directory. This method of pscons rsh to remote nodes only works
# for one pscons in the current working directory, otherwise the
# multiple tasks will write to the same file and make a mess!
# Esteban noticed this in December 2015.
# kls Karl Schleicher This can be trapped by making sure no hosts.txt
# already exists and exiting with an error. hosts.txt will need to be
# deleted at the end. That will be a project for another day!
# YOu cannot append pid to hosts.txt, or the next time you run,
# the command wil be different and scons wilL think it needs to be
# rerun.
if len(self.nodes) > 1:
self.hosts = self.cwd + '/hosts.txt'
print "using %s to keep track of nodes in use by pscons."%self.hosts
# I would like to delete hosts.txt file at end of scons. Cannot just
# delete in def End(self). Need to do after all the scons command
# complete. kls
if (os.path.isfile(self.hosts)):
# would like to delete at end and make error message here it exists.
os.unlink(self.hosts)
hosts_fd=open(self.hosts,'w')
hosts_fd.write("numnodes %4d\n"%len(self.nodes))
hosts_fd.write("host state\n")
for host in self.nodes:
hosts_fd.write(string.ljust(host,40)+string.ljust("notrunning",10)+"\n")
hosts_fd.close()
for key in self['ENV'].keys():
# quote the env values because stampede has env variable
# SLURM_NODE_ALIASES=(null) This makes problems in the ssh to nodes.
# on stampede there are many env keys starting with SLURM_ or
# TACC_ that include list of nodes or queue_job_number. There
# will make pscons rerun the tasks. Some are harmless but
# clutter the printed job output. They can be safely removed
if (key[:6]!='SLURM_' and
key[:5]!='TACC_' and
key[:12]!='_ModuleTable'):
self.environ = self.environ + " %s='%s'" %(key,self['ENV'][key])
def __init__(self, **kw):
kw.update({"tools": []})
apply(Environment.__init__, (self,), kw)
rsf.conf.set_options(self)
# sourceforge = 'http://sourceforge.net/p/rsf/code/HEAD/tree/trunk'
github = "https://github.com/ahay/src/blob/master/"
self.Append(
ENV={
"XAUTHORITY": os.path.join(os.environ.get("HOME"), ".Xauthority"),
"DISPLAY": os.environ.get("DISPLAY"),
"RSF_REPOSITORY": os.environ.get("RSF_REPOSITORY", github),
"RSF_ENSCRIPT": WhereIs("enscript"),
"HOME": os.environ.get("HOME"),
},
SCANNERS=LaTeXS,
BUILDERS={"Latify": Latify, "Pdf": Pdf, "Wiki": Wiki, "Build": Build, "Color": Color, "Figs": Figs},
)
path = {"darwin": ["/sw/bin", "/opt/local/bin"], "irix": ["/usr/freeware/bin"]}
for plat in path.keys():
if sys.platform[: len(plat)] == plat:
for pathdir in filter(os.path.isdir, path[plat]):
self["ENV"]["PATH"] = ":".join([pathdir, self["ENV"]["PATH"]])
tree = rsf.path.dirtree()
root = self.get("RSFROOT", rsf.prog.RSFROOT)
self.docdir = os.environ.get("RSFBOOK", os.path.join(root, "share", "madagascar"))
self.figdir = os.environ.get("RSFFIGS", os.path.join(self.docdir, "figs"))
for level in tree:
if level:
self.docdir = os.path.join(self.docdir, level)
rsf.path.mkdir(self.docdir)
datapath = rsf.path.datapath()
self.path = os.path.dirname(datapath)
if datapath[:2] != "./":
for level in tree[1:]:
self.path = os.path.join(self.path, level)
rsf.path.mkdir(self.path)
self.path = os.path.join(self.path, os.path.basename(datapath))
rsf.path.sconsign(self)
if pdfread:
self.Append(BUILDERS={"Read": Read, "Print": Print})
if epstopdf:
self.Append(BUILDERS={"PDFBuild": PDFBuild})
if fig2dev:
self.Append(BUILDERS={"XFig": XFig})
if latex2html:
self.Append(BUILDERS={"HTML": HTML, "PNGBuild": PNGBuild})
self.imgs = []
if (acroread and ps2eps) or pdf2ps:
self.Append(BUILDERS={"PSBuild": PSBuild})
if epstopdf:
if mathematica:
self.Append(BUILDERS={"Math": Math})
if gnuplot:
self.Append(BUILDERS={"Gnuplot": Gnuplot})
if matlab:
self.Append(BUILDERS={"Matlab": Matlab})
if haspylab:
self.Append(BUILDERS={"Pylab": Pylab})
if sage:
self.Append(BUILDERS={"Sage": Sage})
self.scons = []
self.figs = []
self.Dir()
def __init__(self,**kw):
kw.update({'tools':[]})
apply(Environment.__init__,(self,),kw)
rsf.conf.set_options(self)
# sourceforge = 'http://sourceforge.net/p/rsf/code/HEAD/tree/trunk'
github = 'https://github.com/ahay/src/blob/master/'
self.Append(ENV={'XAUTHORITY':
os.path.join(os.environ.get('HOME'),'.Xauthority'),
'DISPLAY': os.environ.get('DISPLAY'),
'RSF_REPOSITORY': os.environ.get('RSF_REPOSITORY',github),
'RSF_ENSCRIPT': WhereIs('enscript'),
'HOME': os.environ.get('HOME')},
SCANNERS=LaTeXS,
BUILDERS={'Latify':Latify,
'Pdf':Pdf,
'Wiki':Wiki,
'Build':Build,
'Color':Color,
'Figs':Figs})
path = {'darwin': ['/sw/bin','/opt/local/bin'],
'irix': ['/usr/freeware/bin']}
for plat in path.keys():
if sys.platform[:len(plat)] == plat:
for pathdir in filter(os.path.isdir,path[plat]):
self['ENV']['PATH'] = ':'.join([pathdir,
self['ENV']['PATH']])
tree = rsf.path.dirtree()
root = self.get('RSFROOT',rsf.prog.RSFROOT)
self.docdir = os.environ.get('RSFBOOK',os.path.join(root,'share','madagascar'))
self.figdir = os.environ.get('RSFFIGS',os.path.join(self.docdir,'figs'))
for level in tree:
if level:
self.docdir = os.path.join(self.docdir,level)
rsf.path.mkdir(self.docdir)
datapath = rsf.path.datapath()
self.path = os.path.dirname(datapath)
if datapath[:2] != './':
for level in tree[1:]:
self.path = os.path.join(self.path,level)
rsf.path.mkdir(self.path)
self.path = os.path.join(self.path,os.path.basename(datapath))
rsf.path.sconsign(self)
if pdfread:
self.Append(BUILDERS={'Read':Read,'Print':Print})
if epstopdf:
self.Append(BUILDERS={'PDFBuild':PDFBuild})
if fig2dev:
self.Append(BUILDERS={'XFig':XFig})
if latex2html:
self.Append(BUILDERS={'HTML':HTML,'PNGBuild':PNGBuild})
self.imgs = []
if (acroread and ps2eps) or pdf2ps:
self.Append(BUILDERS={'PSBuild':PSBuild})
if epstopdf:
if mathematica:
self.Append(BUILDERS={'Math':Math})
if gnuplot:
self.Append(BUILDERS={'Gnuplot':Gnuplot})
if matlab:
self.Append(BUILDERS={'Matlab':Matlab})
if haspylab:
self.Append(BUILDERS={'Pylab':Pylab})
if sage:
self.Append(BUILDERS={'Sage':Sage})
self.scons = []
self.figs = []
self.Dir()
def __init__(self,**kw):
apply(Environment.__init__,(self,),kw)
# self.EnsureSConsVersion(0,96)
opts = {
'TIMER':'Whether to time execution',
'CHECKPAR':'Whether to check parameters',
'ENVIRON':'Additional environment settings',
'CLUSTER':'Nodes available on a cluster',
'BATCH' : 'Parameter file for batch jobs on a cluster'
}
rsf.conf.set_options(self,opts)
root = self.get('RSFROOT',rsf.prog.RSFROOT)
self.bindir = os.path.join(root,'bin')
self.sfpen = os.path.join(self.bindir,'sfpen')
self.pspen = os.path.join(self.bindir,'pspen')
self.vppen = os.path.join(self.bindir,'vppen')
self.runonnode = os.path.join(self.bindir,'runonnode')
self.figs = os.environ.get('RSFFIGS',os.path.join(root,'share','madagascar','figs'))
cwd = os.getcwd()
self.cwd = cwd
# path for binary files
self.path = rsf.path.getpath(cwd)
tmpdatapath = os.environ.get('TMPDATAPATH',self.path)
rsf.path.sconsign(self)
self.resdir = resdir
self.figdir = re.sub('.*\/((?:[^\/]+)\/(?:[^\/]+)\/(?:[^\/]+))$',
self.figs+'/\\1',cwd)
self.progsuffix = self['PROGSUFFIX']
# Keep certain environmental variables in the environment
for env in keepenv:
getenv = os.environ.get(env)
if getenv:
self.Append(ENV={env:getenv})
# Keep environmental variables needed for SLURM
for env in os.environ.keys():
if 'SLURM_' == env[:6] or 'TACC_' == env[:5] or '_ModuleTable' == env[:12]:
self.Append(ENV={env:os.environ[env]})
if sys.platform[:6] == 'cygwin':
exe = ''
else:
exe = '.exe'
libdir = os.path.join(root,'lib')
incdir = os.path.join(root,'include')
self.Append(ENV={'RSFROOT':root,
'DATAPATH':self.path,
'OMP_NUM_THREADS': os.environ.get('OMP_NUM_THREADS',rsf.node.cpus()),
'TMPDATAPATH': tmpdatapath,
'XAUTHORITY':
os.environ.get('XAUTHORITY',
os.path.join(os.environ.get('HOME'),
'.Xauthority'))},
BUILDERS={'Retrieve':Retrieve,
'Test':Test,
'Echo':Echo},
LIBPATH=[libdir],
CPPPATH=[incdir],
F90PATH=[incdir],
LIBS=[libs],
PROGSUFFIX=exe)
self.Prepend(LIBS=[self.get('DYNLIB','')+'rsf'])
minesjtk = self.get('MINESJTK',None)
usejava = self.get('JAVA_HOME',None)
if not usejava: usejava = self.get('JAVA_SDK',None)
if minesjtk or usejava:
classpath = []
classpath.append(os.path.join(libdir,'rsf.jar'))
userclasspath = os.environ.get('CLASSPATH',None)
if userclasspath: classpath.append(userclasspath)
if minesjtk:
classpath.append(minesjtk)
self.Append(JAVACLASSPATH=':'.join(classpath))
classpath.append('.')
self.Append(ENV={'CLASSPATH':':'.join(classpath)})
path = {'darwin': '/opt/local/bin',
'irix': '/usr/freeware/bin',
'cygwin': '/usr/X11R6/bin:/usr/lib/lapack'}
for plat in path.keys():
if sys.platform[:len(plat)] == plat:
self['ENV']['PATH'] = ':'.join([path[plat],
self['ENV']['PATH']])
pythonpath = os.path.join(sys.prefix,'bin')
if os.path.isdir(pythonpath):
self['ENV']['PATH'] = ':'.join([pythonpath,
self['ENV']['PATH']])
if sys.platform[:6] == 'cygwin':
self['ENV']['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
self['PROGPREFIX']=''
self.view = []
self.prnt = []
self.lock = []
self.test = []
self.coms = []
self.data = []
self.rest = []
sys.path.append('../../../Recipes')
timer = self.get('TIMER')
if timer and timer[0] != 'n' and timer[0] != '0':
self.timer = WhereIs('time') + ' '
else:
self.timer = ''
self.mpirun = self.get('MPIRUN',WhereIs('ibrun') or WhereIs('mpirun'))
raddenv = os.environ.get('RSF_RADDENV', None)
if raddenv:
self.raddenv = raddenv
else:
self.raddenv = ''
checkpar = self.get('CHECKPAR')
self.checkpar = checkpar and checkpar[0] != 'n' and checkpar[0] != '0'
self.environ = self.get('ENVIRON','')
self.batch = self.get('BATCH')
self.jobs = GetOption('num_jobs') # getting information from scons -j
cluster = self.get('CLUSTER',os.environ.get('RSF_CLUSTER','localhost 1'))
hosts = string.split(cluster)
self.nodes = []
for i in range(1,len(hosts),2):
nh = int(hosts[i])
self.nodes.extend([hosts[i-1]]*nh)
self.ip = 0
# self.nodes is a list of CPUs
# self.jobs is the number of jobs
# self.ip is the current CPU
# Esteban Diaz comment: [email protected]
# This bit of the code conflicts with our configuration at mines
# maybe the file could be named hosts-(uname).txt
# In our cluster config, we have several sconstructs that
# try to write the same file. Hence, it produces problems.
#self.hosts = self.path + 'hosts.txt'
#if (os.path.isfile(self.hosts)):
# os.unlink(self.hosts)
#hosts_fd=open(self.hosts,'w')
#hosts_fd.write("numnodes %4d\n"%len(self.nodes))
#hosts_fd.write("host state\n")
#for host in self.nodes:
# hosts_fd.write(string.ljust(host,40)+string.ljust("notrunning",10)+"\n")
#hosts_fd.close()
for key in self['ENV'].keys():
# quote the env values because stampede has env variable
# SLURM_NODE_ALIASES=(null) This makes problems in the ssh to nodes.
self.environ = self.environ + " %s='%s'" % (key,self['ENV'][key])
def __init__(self, **kw):
apply(Environment.__init__, (self, ), kw)
# self.EnsureSConsVersion(0,96)
opts = {
'TIMER': 'Whether to time execution',
'CHECKPAR': 'Whether to check parameters',
'ENVIRON': 'Additional environment settings',
'CLUSTER': 'Nodes available on a cluster',
'BATCH': 'Parameter file for batch jobs on a cluster'
}
rsf.conf.set_options(self, opts)
root = self.get('RSFROOT', rsf.prog.RSFROOT)
self.bindir = os.path.join(root, 'bin')
self.sfpen = os.path.join(self.bindir, 'sfpen')
self.pspen = os.path.join(self.bindir, 'pspen')
self.vppen = os.path.join(self.bindir, 'vppen')
self.runonnode = os.path.join(self.bindir, 'runonnode')
self.figs = os.environ.get(
'RSFFIGS', os.path.join(root, 'share', 'madagascar', 'figs'))
cwd = os.getcwd()
self.cwd = cwd
# path for binary files
self.path = rsf.path.getpath(cwd)
tmpdatapath = os.environ.get('TMPDATAPATH', self.path)
rsf.path.sconsign(self)
self.resdir = resdir
self.figdir = re.sub('.*\/((?:[^\/]+)\/(?:[^\/]+)\/(?:[^\/]+))$',
self.figs + '/\\1', cwd)
self.progsuffix = self['PROGSUFFIX']
# Keep certain environmental variables in the environment
for env in keepenv:
getenv = os.environ.get(env)
if getenv:
self.Append(ENV={env: getenv})
self.hostname = socket.gethostname()
# Keep environmental variables needed for SLURM
for env in os.environ.keys():
if 'SLURM_' == env[:
6] or 'TACC_' == env[:
5] or '_ModuleTable' == env[:
12]:
self.Append(ENV={env: os.environ[env]})
if sys.platform[:6] == 'cygwin':
exe = ''
else:
exe = '.exe'
libdir = os.path.join(root, 'lib')
incdir = os.path.join(root, 'include')
self.Append(ENV={
'RSFROOT':
root,
'DATAPATH':
self.path,
'OMP_NUM_THREADS':
os.environ.get('OMP_NUM_THREADS', rsf.node.cpus()),
'TMPDATAPATH':
tmpdatapath,
'XAUTHORITY':
os.environ.get('XAUTHORITY',
os.path.join(os.environ.get('HOME'), '.Xauthority'))
},
BUILDERS={
'Retrieve': Retrieve,
'Test': Test,
'Echo': Echo
},
LIBPATH=[libdir],
CPPPATH=[incdir],
F90PATH=[incdir],
LIBS=[libs],
PROGSUFFIX=exe)
self.Prepend(LIBS=[self.get('DYNLIB', '') + 'rsf'])
minesjtk = self.get('MINESJTK', None)
usejava = self.get('JAVA_HOME', None)
if not usejava: usejava = self.get('JAVA_SDK', None)
if minesjtk or usejava:
classpath = []
classpath.append(os.path.join(libdir, 'rsf.jar'))
userclasspath = os.environ.get('CLASSPATH', None)
if userclasspath: classpath.append(userclasspath)
if minesjtk:
classpath.append(minesjtk)
self.Append(JAVACLASSPATH=':'.join(classpath))
classpath.append('.')
self.Append(ENV={'CLASSPATH': ':'.join(classpath)})
path = {
'darwin': '/opt/local/bin',
'irix': '/usr/freeware/bin',
'cygwin': '/usr/X11R6/bin:/usr/lib/lapack'
}
for plat in path.keys():
if sys.platform[:len(plat)] == plat:
self['ENV']['PATH'] = ':'.join(
[path[plat], self['ENV']['PATH']])
pythonpath = os.path.join(sys.prefix, 'bin')
if os.path.isdir(pythonpath):
self['ENV']['PATH'] = ':'.join([pythonpath, self['ENV']['PATH']])
if sys.platform[:6] == 'cygwin':
self['ENV']['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
self['PROGPREFIX'] = ''
self.view = []
self.prnt = []
self.lock = []
self.test = []
self.coms = []
self.data = []
self.rest = []
sys.path.append('../../../Recipes')
timer = self.get('TIMER')
if timer and timer[0] != 'n' and timer[0] != '0':
self.timer = WhereIs('time') + ' '
else:
self.timer = ''
self.mpirun = self.get('MPIRUN', WhereIs('ibrun') or WhereIs('mpirun'))
raddenv = os.environ.get('RSF_RADDENV', None)
if raddenv:
self.raddenv = raddenv
else:
self.raddenv = ''
checkpar = self.get('CHECKPAR')
self.checkpar = checkpar and checkpar[0] != 'n' and checkpar[0] != '0'
self.environ = self.get('ENVIRON', '')
self.batch = self.get('BATCH')
self.jobs = GetOption('num_jobs') # getting information from scons -j
cluster = self.get('CLUSTER',
os.environ.get('RSF_CLUSTER', 'localhost 1'))
hosts = string.split(cluster)
self.nodes = []
for i in range(1, len(hosts), 2):
nh = int(hosts[i])
self.nodes.extend([hosts[i - 1]] * nh)
self.ip = 0
# self.nodes is a list of CPUs
# self.jobs is the number of jobs
# self.ip is the current CPU
# Karl Schleicher kls. I'll move hosts.txt back to the current working
# directory. This method of pscons rsh to remote nodes only works
# for one pscons in the current working directory, otherwise the
# multiple tasks will write to the same file and make a mess!
# Esteban noticed this in December 2015.
# kls Karl Schleicher This can be trapped by making sure no hosts.txt
# already exists and exiting with an error. hosts.txt will need to be
# deleted at the end. That will be a project for another day!
# YOu cannot append pid to hosts.txt, or the next time you run,
# the command wil be different and scons wilL think it needs to be
# rerun.
if len(self.nodes) > 1:
self.hosts = self.cwd + '/hosts.txt'
print "using %s to keep track of nodes in use by pscons." % self.hosts
# I would like to delete hosts.txt file at end of scons. Cannot just
# delete in def End(self). Need to do after all the scons command
# complete. kls
if (os.path.isfile(self.hosts)):
# would like to delete at end and make error message here it exists.
os.unlink(self.hosts)
hosts_fd = open(self.hosts, 'w')
hosts_fd.write("numnodes %4d\n" % len(self.nodes))
hosts_fd.write("host state\n")
for host in self.nodes:
hosts_fd.write(
string.ljust(host, 40) + string.ljust("notrunning", 10) +
"\n")
hosts_fd.close()
for key in self['ENV'].keys():
# quote the env values because stampede has env variable
# SLURM_NODE_ALIASES=(null) This makes problems in the ssh to nodes.
# on stampede there are many env keys starting with SLURM_ or
# TACC_ that include list of nodes or queue_job_number. There
# will make pscons rerun the tasks. Some are harmless but
# clutter the printed job output. They can be safely removed
if (key[:6] != 'SLURM_' and key[:5] != 'TACC_'
and key[:12] != '_ModuleTable'):
self.environ = self.environ + " %s='%s'" % (key,
self['ENV'][key])
