def GetExecutable():
configuration = GetConfiguration()
(machine, machineEntry, sourceBaseDir) = simlib.GetLocalEnvironment()
configPath = simlib.BuildPath(simenv.CONFIGS_PATH, configuration)
if not (simlib.FileExists(configPath)):
fatal(
"configuration '%s', which has path '%s' does not exist or is not readable"
% (configuration, configPath))
exe = simlib.BuildPath(simenv.CACTUS_PATH, 'exe',
'cactus_%s' % configuration)
if not (simlib.FileExists(exe)):
fatal(
"Executable %s for configuration %s does not exist or is not readable"
% (exe, configuration))
submitScript = simlib.BuildPath(configPath, "SubmitScript")
if not (simlib.FileExists(submitScript)):
warning("empty submit script for configuration %s" % configuration)
submitScript = None
runScript = simlib.BuildPath(configPath, "RunScript")
if not (simlib.FileExists(runScript)):
fatal("Error: empty/missing run script for configuration %s" %
configuration)
return (exe, submitScript, runScript)
def CopyFileWithCaching(srcfile, destdir, cachedir):
"""Copy 'srcfile' into 'destdir' using and maintaining 'cacheDir' as a hard link cache."""
# os.link == create hard link.
# os.makedirs() == recursively make directories.
if not (os.path.exists(cachedir)):
try:
os.makedirs(cachedir)
except:
fatal("could not create cache directory: %s" % cachedir)
filename = simlib.BaseName(srcfile)
cachefile = simlib.BuildPath(cachedir, filename)
dstfile = simlib.BuildPath(destdir, filename)
if not (os.path.exists(cachefile)):
if os.path.exists(dstfile):
try:
os.remove(dstfile)
except (Exception, EnvironmentError), e:
fatal("Could not remove existing destination file %s, %s" %
(dstfile, e))
try:
shutil.copyfile(srcfile, dstfile)
mode = os.stat(srcfile).st_mode
os.chmod(dstfile, mode)
except:
fatal("Could not copy %s to %s" % (srcfile, dstfile))
try:
os.link(dstfile, cachefile)
# If the system creates a symlink instead, copy the file.
if os.path.islink(cachefile):
try:
os.remove(cachefile)
except (Exception, EnvironmentError), e:
fatal(
"Could not remove existing symlink cache file %s, %s" %
(cachefile, e))
raise IOError
except:
# If there is an error, copy the file. It could be that
# the maximum number of hard links per file has been
# reached.
shutil.copyfile(dstfile, cachefile)
mode = os.stat(dstfile).st_mode
os.chmod(cachefile, mode)
return
def GetRemoteExePath(self, local_path, remotes):
exe = simenv.EXECUTABLE
if simenv.OptionsManager.HasOption("remotecactuspath"):
rcdir = simenv.OptionsManager.GetOption("remotecactuspath")
warning("remote cactus path override, using %s" % rcdir)
local_path = simenv.CACTUS_PATH
remotes = rcdir
self.RemotePath = rcdir
else:
self.RemotePath = simlib.ReplaceLeadingPath(
simenv.CACTUS_PATH, simlib.GetLocalSourceBaseDir(), remotes)
# if locals has a double //, trim off the first one.
if local_path.startswith(r"//"):
local_path = local_path[1:]
if exe[0] == "/":
exe = simlib.ReplaceLeadingPath(exe, local_path, remotes)
else:
path = simlib.GetDirSuffix(local_path)
exe = simlib.BuildPath(remotes, path, exe)
return exe
def CheckSyntax(self):
# first, verify the correctness of our mdb syntax file.
sc = pyini.IniSyntaxChecker(self.DescriptionSyntaxFile, self.SyntaxFile)
sc.SyntaxCheck()
# now verify the correctness of our passed in mdb database
self.MachineParser = pyini.IniParser()
for ff in os.listdir(self.mdbDirectory):
if not(ff.endswith(".ini")):
continue
filePath = simlib.BuildPath(simenv.MDB_PATH, ff)
self.MachineParser.UpdateFromIni(filePath, True)
# load the cdb database, which at the moment has no syntax file and convert any blocks to lists
self.ConfigParser = pyini.IniParser(self.cdbFilename)
self.ConvertBlocks()
if self.udbFilename != None:
# import new sections to the machine database, but not the config database.
self.MachineParser.UpdateFromIni(self.udbFilename, True)
self.ConfigParser.UpdateFromIni(self.udbFilename)
syntaxChecker = pyini.IniSyntaxChecker(self.SyntaxFile, self.MachineParser, True)
syntaxChecker.SyntaxCheck()
self.MachineParser.UpdateFromDict(simenv.OptionsManager.MDBKeys)
def list_configurations():
#display("list_configurations")
simlib.RequireMachine()
configs = simlib.GetConfigurations()
configs.sort()
if simenv.OptionsManager.GetOption('name-only'):
for config in configs:
print config
return
if len(configs) == 0:
display("There are no configurations")
return
if simenv.VERBOSE:
display("Configurations:")
for config in configs:
display(" %-40s" % config, noNewLine=True)
exe = simlib.BuildPath(simenv.CACTUS_PATH, 'exe', 'cactus_%s' % config)
if not (os.path.exists(exe)):
display(" [incomplete]")
else:
statinfo = os.stat(exe)
tinfo = time.localtime(statinfo.st_mtime)
date = "%s-%02d-%02d %2d:%02d:%02d" % (
tinfo.tm_year, tinfo.tm_mon, tinfo.tm_mday, tinfo.tm_hour,
tinfo.tm_min, tinfo.tm_sec)
display(" [built %s]" % date)
def HasActiveRestart(sim):
DefineDatabase = simsubs.DefineDatabase()
DefineDatabase.Set("USER", simenv.LocalMachineEntry.user)
simdir = DefineDatabase.SubAll(
simlib.BuildPath(simenv.LocalMachineEntry.basedir, sim))
for elem in os.listdir(simdir):
if elem.endswith("-active"):
return True
return False
def CreateInternalDirs(internaldir):
roots = ['exe', 'cfg', 'run', 'par', 'data']
mdirs = []
for root in roots:
fullpath = simlib.BuildPath(internaldir, root)
try:
os.makedirs(fullpath)
except OSError, e:
fatal("Error: could not make %s directory \"%s\", %s" %
(root, fullpath, e))
mdirs.append(fullpath)
def ini_initializeDestination(self, value):
fullPath = simlib.BuildPath(simenv.BASE_PATH, value)
self.ini_FullPath = fullPath
if self.ParentTree != None:
if self.ParentTree.DestType == 'ini':
if self.ParentTree.ini_FullPath == self.ini_FullPath:
self.ini_DestParser = self.ParentTree.ini_DestParser
return
self.ini_DestParser = pyini.IniParser(self.ini_FullPath)
def BuildVirtual(configName):
display("Building virtual config %s" % configName)
if not (simenv.OptionsManager.HasOption("virtual-executable")):
fatal("virtual executable is not set")
vexe = simenv.OptionsManager.GetOption("virtual-executable")
if not (os.path.exists(vexe)):
fatal("virtual executable %s does not exist or is not readable" % vexe)
newvpath = simlib.BuildPath(simenv.EXE_PATH, "cactus_%s" % configName)
display("Copying virtual executable from %s to %s" % (vexe, newvpath))
simlib.MakeDirs(simenv.EXE_PATH)
shutil.copy(vexe, newvpath)
def GetRestartsForSimulation(self, srcPath, simulationName):
simpath = simlib.BuildPath(srcPath, simulationName)
files = self.Driver.list(simpath)
restarts = list()
for line in files:
if not (line.startswith("output-")):
continue
id = line.split("-")[1]
#dprint("found restart id: %s" % id)
restarts.append(id)
return restarts
def setupTree(self, tree=None):
if tree != None:
self.IniTree = tree
if self.IniTree == None:
fatal("no decision tree defined.")
fullFile = simlib.BuildPath(simenv.LIB_PATH, "dt",
"%s.ini" % self.IniTree)
if not (os.path.exists(fullFile)):
fatal("could not open decision tree %s for reading." % fullFile)
info("decision tree ini: %s" % fullFile)
self.Parser = pyini.IniParser(fullFile)
self.loadConfiguration()
def ExecuteCommand(self,
command,
parrotArguments=False,
stripArguments=None,
sshargs=''):
DefineDatabase = simsubs.DefineDatabase()
cmdargs = ''
if parrotArguments:
for aa in simenv.OptionsManager.args:
cmdargs = '%s %s' % (cmdargs, simlib.QuoteSafe(aa))
argStr = simenv.OptionsManager.BuildOptionString(stripArguments)
command = "%s %s %s" % (command, cmdargs, argStr)
# lets append --machine <remotemachinename> to the arg string to avoid any confusion.
command = "%s --machine=%s" % (command, self.machineName)
cmd = simlib.GetSSHCommand(simenv.LocalMachine, self.machineName,
command, sshargs, self.RemotePath)
machineEntry = simenv.ConfigurationDatabase.GetMachine(
self.machineName)
localSourceBaseDir = simlib.GetLocalSourceBaseDir()
source_name = simlib.GetDirSuffix(localSourceBaseDir)
local_sourcedir = simlib.BuildPath(localSourceBaseDir, source_name)
DefineDatabase.Set('SOURCEDIR', local_sourcedir)
cmd = DefineDatabase.SubAll(cmd)
DefineDatabase.Set('USER', machineEntry.user)
localsshsetup = DefineDatabase.SubAll(machineEntry.localsshsetup)
cmd = "{ %s; } && { %s; } && %s" % (self.localEnvSetup, localsshsetup,
cmd)
#dprint("Executing Command: %s" % cmd)
ret = simlib.ExecuteCommand(cmd)
(status, signal) = (ret >> 8, ret & 0xff)
if status == 0xff:
fatal("Could not execute command \"%s\"" % cmd)
def buildMetadataFile(self):
# for now, lets just use properties.ini from the restart
return simlib.BuildPath(self.Restart.InternalDir, "properties.ini")
def PrepConfiguration(configName):
DefineDatabase = simsubs.DefineDatabase()
machineEntry = simenv.LocalMachineEntry
req_keys = ['user', 'email', 'make']
simlib.VerifyKeys(machineEntry, req_keys)
user = machineEntry.GetKey("user")
email = machineEntry.GetKey("email")
DefineDatabase.Set('USER', user)
DefineDatabase.Set('EMAIL', email)
make = DefineDatabase.SubAll(machineEntry.GetKey('make'))
subfolders = ['bindings', 'build', 'config-data', 'lib', 'scratch']
configBase = simlib.BuildPath(simenv.CONFIGS_PATH, configName)
propertiesFile = simlib.BuildPath(configBase, "properties.ini")
if not os.path.exists(configBase):
try:
os.mkdir(configBase)
except OSError:
pass
#simlib.ExecuteCommand("mkdir %s" % configBase)
for sub in subfolders:
sf = simlib.BuildPath(configBase, sub)
simlib.ExecuteCommand("mkdir %s" % sf)
else: # The config dir exists already
if not os.path.exists(propertiesFile):
fatal(
"Configuration %s has no properties.ini file. Either this is not a SimFactory configuration, or the configuration has become corrupted. If you think the configuration has been corrupted, you will need to delete it and run the sim build command again."
% (configName))
prop = simproperties.SimProperties()
prop.init(propertiesFile)
build_reconfig = GetConfigValue(prop, "reconfig")
build_clean = GetConfigValue(prop, "clean")
build_debug = GetConfigValue(prop, "debug")
build_optimise = GetConfigValue(prop, "optimise")
build_unsafe = GetConfigValue(prop, "unsafe")
build_profile = GetConfigValue(prop, "profile")
if build_debug:
display("Disabling optimisation because debug was selected")
build_optimise = False
# update our properties.ini
prop.RemoveProperty("reconfig")
prop.RemoveProperty("clean")
prop.AddProperty("debug", build_debug)
prop.AddProperty("optimise", build_optimise)
prop.AddProperty("unsafe", build_unsafe)
prop.AddProperty("profile", build_profile)
# Remove the executable, so that it is not accidentally used
# while it is rebuilt. Do this before the configuration is
# modified in any way, but only after some basic error
# checking.
try:
os.remove("exe/cactus_%s" % configName)
except OSError:
pass
try:
shutil.rmtree("exe/%s" % configName, ignore_errors=True)
except OSError:
pass
prop.Save()
storedOptions = simlib.BuildPath(configBase, "OptionList")
hasStoredOptions = os.path.exists(storedOptions)
cctkConfigPath = simlib.BuildPath(configBase, "config-data",
"cctk_Config.h")
hasCompleteConfig = os.path.exists(cctkConfigPath)
if hasStoredOptions:
storedOptionSettings = simlib.GetFileContents(storedOptions)
else:
storedOptionSettings = None
info("HasStoredOptions: %s" % hasStoredOptions)
removeConfig = False # default behaviour
# we are only updating the OptionList if its explicitly specified with --optionlist, really.
if hasStoredOptions and not simenv.OptionsManager.RawOptionDefined(
"optionlist"):
info("Use stored options list: %s" % storedOptions)
optionlist = storedOptions
else:
optionlist = simlib.GetOptionList(False)
info("optionlist is: %s" % optionlist)
if optionlist == None or not os.path.exists(optionlist):
display("Warning: no option list specified, using blank option list")
optionSettings = str()
else:
optionSettings = simlib.GetFileContents(optionlist)
DefineDatabase.AddReplacement("DEBUG", simlib.BoolToYesNo(build_debug))
DefineDatabase.AddReplacement("OPTIMISE",
simlib.BoolToYesNo(build_optimise))
DefineDatabase.AddReplacement("UNSAFE", simlib.BoolToYesNo(build_unsafe))
DefineDatabase.AddReplacement("PROFILE", simlib.BoolToYesNo(build_profile))
# add support for CACHELINE_BYTES and CACHE_SIZE
CACHELINE_BYTES = simenv.LocalMachineEntry.GetKey("L3linesize")
if CACHELINE_BYTES != None:
DefineDatabase.AddReplacement("CACHELINE_BYTES", CACHELINE_BYTES)
CACHE_SIZE = simenv.LocalMachineEntry.GetKey("L3size")
if CACHE_SIZE != None:
DefineDatabase.AddReplacement("CACHE_SIZE", CACHE_SIZE)
optionSettings = DefineDatabase.SubAll(optionSettings)
hasOutdatedConfig = (not hasCompleteConfig) or (
storedOptionSettings
== None) or (storedOptionSettings != optionSettings)
if hasOutdatedConfig:
info("hasCompleteConfig: %s" % hasCompleteConfig)
info("hasOutdatedConfig: %s" % hasOutdatedConfig)
info("build_reconfig: %s" % build_reconfig)
# if build virtual, disable configging.
if hasOutdatedConfig or build_reconfig:
if storedOptionSettings != None:
oldVersion = simlib.GetVersion(storedOptionSettings)
else:
oldVersion = ""
newVersion = simlib.GetVersion(optionSettings)
info("oldVersion %s, newVersion %s" % (oldVersion, newVersion))
removeConfig = removeConfig or newVersion != oldVersion
display("Reconfiguring %s" % configName)
simlib.RemoveFile("%s.old" % storedOptions)
simlib.RenameFile(storedOptions, "%s.old" % storedOptions)
# TODO: Write new option list only after the make *-realclean below!
display("Writing configuration to: %s" % storedOptions)
simlib.WriteContents(storedOptions, optionSettings)
#"echo yes | { $make $configuration_name-config options=configs/$configuration_name/OptionList; }";
if not simenv.OptionsManager.GetOption('virtual'):
cmd = "cd %s && echo yes | { %s %s-config options=%s; } 2>&1" % (
simenv.CACTUS_PATH, make, configName, storedOptions)
ret = simlib.ExecuteCommand(cmd)
if ret != 0:
sys.exit(1)
#force a rebuild
simlib.RemoveFile(
simlib.BuildPath(configBase, 'config-data', 'make.thornlist'))
#remove the old config if necessary
if removeConfig:
display("Complete rebuild required")
else:
if not hasStoredOptions:
fatal(
"Configuration %s has no option list, and no option list was specified."
% configName)
if not simenv.OptionsManager.GetOption('virtual'):
info("build_clean: %s" % build_clean)
info("removeConfig: %s" % removeConfig)
if build_clean or removeConfig:
display("Cleaning %s" % configName)
ret = simlib.ExecuteCommand("cd %s && %s %s-realclean 2>&1" %
(simenv.CACTUS_PATH, make, configName))
if ret != 0:
sys.exit(1)
## deal with submit script now
storedSubmitScript = simlib.BuildPath(configBase, "SubmitScript")
hasStoredSubmitScript = os.path.exists(storedSubmitScript)
if simenv.OptionsManager.GetOption('no-submitscript'):
if hasStoredSubmitScript:
display("Removing stored submit script for configuration %s" %
configName)
os.unlink(storedSubmitScript)
warning("empty submit script will disable submission")
else:
# the submit script is entirely optional. no submit script
# just means submission is disabled.
submitscript = simlib.GetSubmitScript()
info("SubmitScript is: %s" % submitscript)
if submitscript != None:
sContents = simlib.GetFileContents(submitscript)
ssContents = simlib.GetFileContents(storedSubmitScript)
submitScriptOutdated = not hasStoredSubmitScript or simenv.OptionsManager.RawOptionDefined(
"submitscript")
if sContents != ssContents:
warning("default submit script contents have changed")
if (submitScriptOutdated or build_reconfig):
display("Updated script file for configuration %s" %
configName)
simlib.RemoveFile("%s.old" % storedSubmitScript)
simlib.RenameFile(storedSubmitScript,
"%s.old" % storedSubmitScript)
shutil.copy(submitscript, storedSubmitScript)
else:
if not hasStoredSubmitScript:
warning("empty submit script will disable submission")
## deal with run script now
storedRunScript = simlib.BuildPath(configBase, "RunScript")
hasStoredRunScript = os.path.exists(storedRunScript)
runscript = simlib.GetRunScript(False)
info("RunScript is: %s" % runscript)
if runscript != None:
sContents = simlib.GetFileContents(runscript)
ssContents = simlib.GetFileContents(storedRunScript)
runScriptOutdated = not hasStoredRunScript or simenv.OptionsManager.RawOptionDefined(
"runscript")
if sContents != ssContents:
warning("default run script contents have changed")
if (runScriptOutdated or build_reconfig):
display("Updated runscript file for configuration %s" % configName)
simlib.RemoveFile("%s.old" % storedRunScript)
simlib.RenameFile(storedRunScript, "%s.old" % storedRunScript)
shutil.copy(runscript, storedRunScript)
else:
if not hasStoredRunScript:
fatal(
"Configuration %s has no run script, and no run script file was specified"
% configName)
## deal with thorn list now
storedThornList = simlib.BuildPath(configBase, "ThornList")
hasStoredThornList = os.path.exists(storedThornList)
needThornList = not hasStoredThornList
thornlist = simlib.GetThornList(needThornList)
info("ThornList is: %s" % thornlist)
if thornlist != None:
tContents = simlib.GetThornListContents(thornlist)
ttContents = simlib.GetFileContents(storedThornList)
thornListOutdated = not hasStoredThornList or simenv.OptionsManager.RawOptionDefined(
"thornlist")
if tContents != ttContents:
warning("default thorn list contents have changed")
if (thornListOutdated or build_reconfig):
display("Updated thorn list for configuration %s" % configName)
simlib.RemoveFile("%s.old" % storedThornList)
simlib.RenameFile(storedThornList, "%s.old" % storedThornList)
simlib.WriteContents(storedThornList, tContents)
else:
if not hasStoredThornList:
fatal(
"Configuration %s has no thorn list, and no thorn list file was specified"
% configName)
def CompileCommand(machineName, rsyncInfo, paths, oldRsync=False):
global localMachineName
global localMachineEntry
global local_sourcebasedir
global local_suffix
DefineDatabase = simsubs.DefineDatabase()
(rsynccmd, rsyncopts) = rsyncInfo
rsyncversion = simlib.RsyncVersion(rsyncInfo)
if machineName == localMachineName:
fatal("cannot sync to local machine")
machineEntry = simenv.ConfigurationDatabase.GetMachine(machineName)
# get our IO machine -- if iomachine doesn't exist, it returns itself.
ioMachine = simlib.GetIOMachine(machineName)
if ioMachine != machineName:
machineEntry = simenv.ConfigurationDatabase.GetMachine(ioMachine)
machineName = ioMachine
# make sure we have all the info we need.
mreq_keys = [
'user', 'sourcebasedir', 'hostname', 'rsynccmd', 'rsyncopts', 'sshcmd',
'sshopts'
]
simlib.VerifyKeys(machineEntry, mreq_keys)
DefineDatabase.Set('USER', machineEntry.user)
sourcebasedir = DefineDatabase.SubAll(machineEntry.sourcebasedir)
# There need to be two define databases, one for the local and
# another for the remote system. Currently, USER is for the
# remote and SOURCEDIR is for the local system -- this is
# inconsistent.
#local_sourcebasedir = DefineDatabase.SubAll(localMachineEntry.sourcebasedir)
source_name = local_suffix
#source_name = simlib.GetDirSuffix(local_sourcebasedir)
local_sourcedir = simlib.BuildPath(local_sourcebasedir, source_name)
DefineDatabase.Set('SOURCEDIR', local_sourcedir)
localsshsetup = DefineDatabase.SubAll(machineEntry.localsshsetup)
# sshcmd = machineEntry.sshcmd
# sshopts = machineEntry.sshopts
# sshcmd = "%s %s" % (sshcmd, sshopts)
# trampoline = simlib.GetTrampoline(ioMachine)
# if trampoline:
# tentry = simenv.ConfigurationDatabase.GetMachine(trampoline)
# trampoline_sourcebasedir = simlib.GetSourceBaseDir(tentry)
# trampoline_sourcedir = os.path.join(trampoline_sourcebasedir, source_name)
# DefineDatabase.Set('SOURCEDIR', trampoline_sourcedir)
# sshcmd = DefineDatabase.SubAll(sshcmd)
# DefineDatabase.Set('SOURCEDIR', local_sourcedir)
# sshcmd = simlib.GetSSHCommand(trampoline, sshcmd)
# sshcmd = DefineDatabase.SubAll(sshcmd)
sshcmd = simlib.GetSSHCommand(localMachineName, ioMachine, None)
# If there is more than one explicit path given only accept
# top-level directories This could be extended to do the right
# thing when multiple explicit paths are given but would require
# running rsync multiple times
rsyncfiles = []
if len(paths) == 1:
rsyncfiles = paths
[head, tail] = os.path.split(rsyncfiles[0])
local_suffix = os.path.join(local_suffix, head)
else:
for file in paths:
if file == '':
continue
[head, tail] = os.path.split(file)
if head != "":
fatal(
"Only top level paths may be specified when syncing multiple paths but '%s' given."
% file)
elif not os.path.exists(tail):
warning("Specified sync path '%s' not found." % tail)
else:
rsyncfiles.append(tail)
# Note:
# - We omit --times, since the target system should recompile when
# a file changed there
# - We omit --group and --owner, since this probably works for
# root only, and all files on the targe system should rather
# belong to the user
# - We omit -D, since we don't expect to have devices or other
# special files
# - Since we omit --times, we add --checksum so that differing
# modification times don't lead to a retransmission
# - NOTE: rsync before 3.0.8 has an error that makes --checksum
# unreliable under certain circumstances (see
# <https://rsync.samba.org/ftp/rsync/src/rsync-3.0.8-NEWS>). We
# cannot use --ignore-times instead because this modifies all
# files' timestamps (and is also very expensive).
rsyncoptions = [
'--checksum',
'--compress',
'--delete',
'--hard-links',
'--links',
'--partial',
'--perms',
'--progress',
'--recursive',
'--sparse',
'--stats',
#'--times',
'--verbose'
]
# We need to use a different rules file for pre 3.0.0 versions of rsync
ruleFile = 'filter.rules'
if rsyncversion[0] < 3 or oldRsync:
info(
'Old rsync version in use (local version is %s.%s.%s). Upgrade to at least 3.0.0 both locally and remotely for best performance.'
% rsyncversion)
ruleFile = 'filter.prersync3.rules'
userRuleFile = "%s/etc/%s" % (simenv.BASE_PATH, "filter.local.rules")
if os.path.exists(userRuleFile):
rsyncoptions.append("--filter 'merge %s'" % (userRuleFile))
rsyncoptions.append("--filter 'merge %s/etc/%s'" %
(simenv.BASE_PATH, ruleFile))
fullpath = "%s@%s:%s%s%s" % (machineEntry.user, machineEntry.hostname,
sourcebasedir, os.sep, local_suffix)
arguments = " ".join(simlib.GetArguments())
cmd = "%s --rsh=%s --rsync-path=%s %s %s %s" % (
rsynccmd, simlib.QuoteSafe(sshcmd),
simlib.QuoteSafe(machineEntry.rsynccmd), rsyncopts,
machineEntry.rsyncopts, arguments)
cmd = "%s %s" % (cmd, " ".join(rsyncoptions))
cmd = "%s %s" % (cmd, " ".join(rsyncfiles))
cmd = "%s %s" % (cmd, fullpath)
mkdirpath = os.path.join(sourcebasedir, local_suffix)
mkdircmd = "mkdir -p %s" % simlib.QuoteSafe(mkdirpath)
#mkdircmd = "%s %s@%s %s" % (sshcmd, machineEntry.user, machineEntry.hostname, simlib.QuoteSafe(mkdircmd))
mkdircmd = simlib.GetSSHCommand(localMachineName, ioMachine, mkdircmd)
cmd = "cd %s && { %s; } && { %s || { %s && %s; } }" % (
simlib.QuoteSafe(local_sourcedir), localsshsetup, cmd, mkdircmd, cmd)
return cmd
def init(base, callingExecutable, usageString, optionGroups):
import simopts
import simdb
import simlib
global_dict = globals()
if global_dict['INITIALIZED']:
return
global_dict['BASE_PATH'] = base
global_dict['LIB_PATH'] = "%s%s%s" % (global_dict['BASE_PATH'], os.sep,
"lib")
global_dict['ETC_PATH'] = "%s%s%s" % (global_dict['BASE_PATH'], os.sep,
"etc")
global_dict['LOG_PATH'] = "%s%s%s" % (global_dict['BASE_PATH'], os.sep,
"../log")
global_dict['MDB_PATH'] = "%s%s%s%s%s" % (global_dict['BASE_PATH'], os.sep,
"mdb", os.sep, "machines")
global_dict['MDB_BASE_PATH'] = "%s%s%s" % (global_dict['BASE_PATH'],
os.sep, "mdb")
global_dict['SYNTAX_PATH'] = "%s%s%s" % (global_dict['ETC_PATH'], os.sep,
"syntax")
global_dict['OPTIONS_PATH'] = "%s%s%s" % (global_dict['ETC_PATH'], os.sep,
"options")
global_dict['CACTUS_PATH'] = None
global_dict['VERBOSE'] = True
global_dict['VERSION'] = "???"
global_dict['StoredVerbose'] = global_dict['VERBOSE']
# a couple of paths that are in the cactus root.
global_dict['CONFIGS_PATH'] = None
global_dict['EXE_PATH'] = None
global_dict['EXECUTABLE'] = callingExecutable
global_dict['COMMAND'] = None
global_dict['SETUP'] = False
global_dict['INTERNALDIRECTORY'] = "SIMFACTORY"
global_dict['VERBOSE'] = True
global_dict['udb'] = None
global_dict['cdb'] = None
try:
fd = os.popen("svnversion %s 2>&1" % global_dict['BASE_PATH'])
ver = fd.read().strip()
fd.close()
#if ver.count(":") > 0:
# global_dict['VERSION'] = "r%s" % ver.split(":").pop(0)
global_dict['VERSION'] = ver
except:
pass
global_dict['OptionsManager'] = simopts.OptionManager(
global_dict['OPTIONS_PATH'])
if optionGroups != None:
for og in optionGroups:
global_dict['OptionsManager'].UseGroup(og)
if usageString != None:
global_dict['OptionsManager'].SetUsage(usageString)
global_dict['OptionsManager'].Build()
global_dict['StoredVerbose'] = global_dict['VERBOSE'] = global_dict[
'OptionsManager'].GetOption('verbose')
# more initialization
GetIniFiles()
# init configuration database
global_dict['ConfigurationDatabase'] = simdb.ConfigurationDatabase(
mdbDirectory=global_dict['MDB_PATH'],
udb=global_dict['udb'],
cdb=global_dict['cdb'])
global_dict['ConfigurationDatabase'].Load()
global_dict['CACTUS_PATH'] = simlib.GetCactusPath()
if global_dict['CACTUS_PATH'] == None:
global_dict['CACTUS_PATH'] = global_dict['BASE_PATH']
print "Warning: Unable to determine CACTUS_PATH, using %s instead" % global_dict[
'CACTUS_PATH']
global_dict['CONFIGS_PATH'] = simlib.BuildPath(
[global_dict['CACTUS_PATH'], "configs"])
if "CACTUS_CONFIGS_DIR" in os.environ:
global_dict['CONFIGS_PATH'] = os.environ["CACTUS_CONFIGS_DIR"]
global_dict['EXE_PATH'] = simlib.BuildPath(
[global_dict['CACTUS_PATH'], "exe"])
# get our local machine, and our local machine entry.
global_dict['LocalMachine'] = simlib.GetMachineName()
if global_dict['LocalMachine'] != None:
global_dict['LocalMachineEntry'] = global_dict[
'ConfigurationDatabase'].GetMachine(global_dict['LocalMachine'])
else:
global_dict['LocalMachineEntry'] = None
cmd = str()
count = 0
for arg in sys.argv:
if count == 0:
cmd = arg
else:
cmd = "%s \"%s\"" % (cmd, arg)
count = count + 1
cmd = cmd.strip()
if global_dict['VERBOSE']:
print "Info: Simfactory command: %s" % cmd
print "Info: Version %s" % global_dict['VERSION']
global_dict['INITIALIZED'] = True
def __init__(self, restart):
self.Restart = restart
self.LogPath = simlib.BuildPath(self.Restart.SimulationDir, "log.txt")
self.LogPointer = None
self.Open()
def macro_GET_BASEDIR(self):
return simlib.BuildPath(os.environ['HOME'], "simulations")
def macro_CREATE_MACHINE(self):
mm = self.DefineDatabase.Get('machine')
srcPath = simlib.BuildPath(simenv.MDB_PATH, 'generic.ini')
dstPath = simlib.BuildPath(simenv.MDB_PATH, '%s.ini' % mm)
shutil.copy(srcPath, dstPath)
self.tParser = pyini.IniParser(dstPath)
self.tParser.parser.RenameSection('generic', mm)
options = ['name', 'hostname', 'nickname']
for oo in options:
io = self.tParser.GetOption(mm, oo)
io.Value = io.value = mm
self.tParser.parser.WriteKey(mm, oo, io)
# sourcebasedir
io = self.tParser.GetOption(mm, "sourcebasedir")
sourcebasedir = self.macro_GET_SOURCEBASEDIR()
io.Value = sourcebasedir
self.tParser.parser.WriteKey(mm, "sourcebasedir", io)
# basedir
io = self.tParser.GetOption(mm, "basedir")
io.Value = self.macro_GET_BASEDIR()
self.tParser.parser.WriteKey(mm, "basedir", io)
# optionlist
if (simenv.OptionsManager.RawOptionDefined("optionlist")):
io = self.tParser.GetOption(mm, "optionlist")
io.Value = simlib.GetOptionList(False)
self.tParser.parser.WriteKey(mm, "optionlist", io)
# submitscript
if (simenv.OptionsManager.RawOptionDefined("submitscript")):
io = self.tParser.GetOption(mm, "submitscript")
io.Value = simlib.GetSubmitScript(False)
self.tParser.parser.WriteKey(mm, "submitscript", io)
# runscript
if (simenv.OptionsManager.RawOptionDefined("runscript")):
io = self.tParser.GetOption(mm, "runscript")
io.Value = simlib.GetRunScript(False)
self.tParser.parser.WriteKey(mm, "runscript", io)
# ppn
if (simenv.OptionsManager.RawOptionDefined("ppn")):
io = self.tParser.GetOption(mm, "ppn")
io.Value = simenv.OptionsManager.GetOption('ppn')
self.tParser.parser.WriteKey(mm, "ppn", io)
# also set 'max-num-threads'
io = self.tParser.GetOption(mm, "max-num-threads")
io.Value = simenv.OptionsManager.GetOption('ppn')
self.tParser.parser.WriteKey(mm, "max-num-threads", io)
# also enable parallel make
io = self.tParser.GetOption(mm, "make")
io.Value = "nice make -j%d" % simenv.OptionsManager.GetOption(
'ppn')
self.tParser.parser.WriteKey(mm, "make", io)
# num-threads
if (simenv.OptionsManager.RawOptionDefined("num-threads")):
io = self.tParser.GetOption(mm, "num-threads")
io.Value = simenv.OptionsManager.GetOption('num-threads')
self.tParser.parser.WriteKey(mm, "num-threads", io)
simlib.WriteContents(dstPath, self.tParser.GetIniAsString())
return "machine %s [%s] created successfully" % (mm, dstPath)
def CreateRestartSkeleton(simulationName):
(machine, machineEntry, sourceBaseDir) = simlib.GetLocalEnvironment()
basedir = simlib.GetBaseDir(machineEntry)
try:
simlib.MakeDirs(basedir)
except OSError, e:
fatal(
"could not access simulation base directory %s for reading and writing: %s"
% (basedir, e))
simulationdir = simlib.BuildPath(basedir, simulationName)
internaldir = simlib.BuildPath(simulationdir, simenv.INTERNALDIRECTORY)
if simlib.FileExists(internaldir):
fatal(
"cannot create job skeleton directory: Simulation \"%s\" already exists and has been initialized"
% simulationdir)
try:
simlib.MakeDirs(simulationdir)
except OSError, e:
fatal("could not create simulation skeleton directory \"%s\", %s" %
(simulationdir, e))
try:
simlib.MakeDirs(internaldir)