def GetThornListContents(filename):
machineEntry = simenv.LocalMachineEntry
contents = GetFileContents(filename, None)
if contents == None:
fatal("could not open file '%s' for reading" % filename)
defineDatabase = simsubs.DefineDatabase()
enabledThorns = (machineEntry.GetKey('enabled-thorns') + "\n" +
machineEntry.GetKey('enabled-thorns-default') + "\n" +
machineEntry.GetKey('enabled-thorns-local') + "\n")
for thorn in enabledThorns.split():
defineDatabase.AddSubstitution(r'^[ \t]*#DISABLED[ \t]+(%s)\b' % thorn,
'@1@')
contents = defineDatabase.SubAll(contents)
defineDatabase = simsubs.DefineDatabase()
disabledThorns = (machineEntry.GetKey('disabled-thorns') + "\n" +
machineEntry.GetKey('disabled-thorns-default') + "\n" +
machineEntry.GetKey('disabled-thorns-local') + "\n")
for thorn in disabledThorns.split():
defineDatabase.AddSubstitution(r'^[ \t]*(%s)\b' % thorn,
'#DISABLED @1@')
contents = defineDatabase.SubAll(contents)
return contents
def ExecuteReplaceCommand(command, output=False):
if (not output) or simenv.VERBOSE:
info("EXECUTING: %s\n" % command)
else:
logonly("EXECUTING: %s\n" % command)
envsetup = GetMachineOption('envsetup')
DefineDatabase = simsubs.DefineDatabase()
DefineDatabase.Set("USER", simenv.LocalMachineEntry.user)
envsetup = DefineDatabase.SubAll(envsetup)
command = "{ %s; } && %s" % (envsetup, command)
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
if pid > 0:
retinfo = os.wait()
return retinfo[1]
else:
args = ['bash', '-c', command]
logonly("executed: /bin/bash -c %s" % " ".join(args))
os.execv('/bin/bash', args)
def BuildConfiguration(configName):
DefineDatabase = simsubs.DefineDatabase()
machineEntry = simenv.LocalMachineEntry
req_keys = ['user', 'email', 'make']
simlib.VerifyKeys(machineEntry, req_keys)
user = machineEntry.GetKey("user")
email = machineEntry.GetKey("email")
DefineDatabase.Set('USER', user)
DefineDatabase.Set('EMAIL', email)
make = DefineDatabase.SubAll(machineEntry.GetKey('make'))
display("Building %s" % configName)
ret = simlib.ExecuteCommand("cd %s && %s %s 2>&1" %
(simenv.CACTUS_PATH, make, configName))
if ret != 0:
sys.exit(1)
display("Building utilities for %s" % configName)
ret = simlib.ExecuteCommand("cd %s && %s %s-utils 2>&1" %
(simenv.CACTUS_PATH, make, configName))
if ret != 0:
sys.exit(1)
def init(self, machineName):
self.localMachineName = simenv.LocalMachine
self.localMachineEntry = simenv.LocalMachineEntry
if self.localMachineEntry == None:
fatal("unknown local machine %s" % self.localMachineName)
self.machineName = machineName
if self.machineName == self.localMachineName:
fatal("remote machine %s is the same as local machine %s" %
(self.machineName, self.localMachineName))
self.machineEntry = simenv.ConfigurationDatabase.GetMachine(
self.machineName)
if self.machineEntry == None:
fatal("unknown remote machine %s" % self.machineName)
req_keys = ['envsetup', 'sourcebasedir', 'user', 'hostname']
simlib.VerifyKeys(self.localMachineEntry, req_keys)
self.localEnvSetup = self.localMachineEntry.GetKey("envsetup")
DefineDatabase = simsubs.DefineDatabase()
DefineDatabase.Set("USER", simenv.LocalMachineEntry.user)
self.localEnvSetup = DefineDatabase.SubAll(self.localEnvSetup)
def command_build():
DefineDatabase = simsubs.DefineDatabase()
configList = simenv.OptionsManager.args
if len(configList) == 0:
configList = [simlib.GetDefaultConfiguration()]
for config in configList:
display("Using configuration: %s" % config)
machineEntry = simenv.LocalMachineEntry
req_keys = ['user', 'email']
simlib.VerifyKeys(machineEntry, req_keys)
user = machineEntry.GetKey("user")
email = machineEntry.GetKey("email")
DefineDatabase.Set('USER', user)
DefineDatabase.Set('EMAIL', email)
cactusDir = simenv.CACTUS_PATH
info("Cactus Directory: %s" % cactusDir)
for config in configList:
PrepConfiguration(config)
if not simenv.OptionsManager.GetOption('virtual'):
BuildConfiguration(config)
else:
BuildVirtual(config)
display("Done.")
def GetSSHCommand(from_machine, machine, cmd, other_opts='', remotePath=None):
if not machine:
fatal('machine is None')
#return cmd
DefineDatabase = simsubs.DefineDatabase()
local_sourcebasedir = GetLocalSourceBaseDir()
sourcename = GetDirSuffix(local_sourcebasedir)
if sourcename == None:
sourcename = ''
# Find out from where we should connect
trampoline = GetTrampoline(machine)
if trampoline != None:
if not simenv.ConfigurationDatabase.HasMachineEntry(trampoline):
fatal('Unknown trampoline "%s" specified for machine "%s"' % (trampoline, machine))
else:
trampoline = from_machine
from_entry = simenv.ConfigurationDatabase.GetMachine(trampoline)
VerifyKeys(from_entry, ['user', 'localsshprefix', 'localsshopts', 'sourcebasedir'])
from_user = from_entry.user
from_sshprefix = from_entry.localsshprefix
from_sshopts = from_entry.localsshopts
from_sourcebasedir = from_entry.sourcebasedir
from_sourcedir = os.path.join(from_sourcebasedir, sourcename)
DefineDatabase.Set('SOURCEDIR', from_sourcedir)
DefineDatabase.Set('USER', from_user)
from_sshprefix = DefineDatabase.SubAll(from_sshprefix)
from_sshopts = DefineDatabase.SubAll(from_sshopts)
entry = simenv.ConfigurationDatabase.GetMachine(machine)
VerifyKeys(entry, ['hostname', 'user', 'sshcmd', 'sshopts', 'sourcebasedir', ])
hostname = entry.hostname
user = entry.user
sshcmd = entry.sshcmd
sshopts = entry.sshopts
sourcebasedir = entry.sourcebasedir
sourcedir = os.path.join(sourcebasedir, sourcename)
DefineDatabase.Set('SOURCEDIR', sourcedir)
DefineDatabase.Set('USER', user)
if cmd != None:
if remotePath != None:
cmd = "cd %s && { %s; }" % (remotePath, cmd)
cmd = "%s@%s %s" % (user, hostname, QuoteSafe(cmd))
else:
cmd = ''
cmd = "%s %s %s %s %s %s" % (from_sshprefix, sshcmd, from_sshopts, sshopts, other_opts, cmd)
cmd = DefineDatabase.SubAll(cmd)
if trampoline != from_machine:
cmd = GetSSHCommand(from_machine, trampoline, cmd, other_opts)
return cmd
def HasActiveRestart(sim):
DefineDatabase = simsubs.DefineDatabase()
DefineDatabase.Set("USER", simenv.LocalMachineEntry.user)
simdir = DefineDatabase.SubAll(
simlib.BuildPath(simenv.LocalMachineEntry.basedir, sim))
for elem in os.listdir(simdir):
if elem.endswith("-active"):
return True
return False
def GetSubbedInfo(machineEntry, keys):
DefineDatabase = simsubs.DefineDatabase()
VerifyKeys(machineEntry, keys)
data = list()
for k in keys:
d = DefineDatabase.SubAll(machineEntry.GetKey(k))
data.append(d)
return data
def ExecuteCommand(command, output=False):
sys.stdout.flush()
sys.stderr.flush()
if command == None or len(command) == 0:
return 0
envsetup = GetMachineOption('envsetup')
DefineDatabase = simsubs.DefineDatabase()
DefineDatabase.Set("USER", simenv.LocalMachineEntry.user)
envsetup = DefineDatabase.SubAll(envsetup)
command = "{ %s; } && { %s; }" % (envsetup, command)
if output:
command = "{ %s; } 2>&1" % command
command = "/bin/bash -c %s" % (QuoteSafe(command))
if (not output) or simenv.VERBOSE:
info("EXECUTING COMMAND: %s\n" % command)
else:
logonly("EXECUTING COMMAND: %s\n" % command)
if not output:
ret = os.system(command)
retval = ret
else:
fd = os.popen(command)
output = fd.read()
ret = fd.close()
if ret == None:
ret = 0
retval = (output, ret)
if ret != 0:
dprint("Error while executing command \"%s\":" % command)
if output:
dprint("Command output: %s" % output)
signal = ret & 0xff
status = ret >> 8
if signal != 0:
dprint(" Command was killed by signal %d" % signal)
else:
dprint(" Command returned exit status %d" % status)
return retval
def GetSourceBaseDir(machineEntry):
DefineDatabase = simsubs.DefineDatabase()
if not(machineEntry.HasKey('user')):
machineName = machineEntry.nickname
fatal("machine entry %s does not have a user defined" % machineName)
user = machineEntry.user
DefineDatabase.Set('USER', user)
sourcebasedir = DefineDatabase.SubAll(machineEntry.sourcebasedir)
if not(sourcebasedir[0] == "/"):
cwd = os.getcwd()
sourcebasedir = "%s%s%s" % (cwd, os.sep, sourcebasedir)
if sourcebasedir.count("//") > 0:
return sourcebasedir[1:]
return sourcebasedir
def ExecuteCommand(self,
command,
parrotArguments=False,
stripArguments=None,
sshargs=''):
DefineDatabase = simsubs.DefineDatabase()
cmdargs = ''
if parrotArguments:
for aa in simenv.OptionsManager.args:
cmdargs = '%s %s' % (cmdargs, simlib.QuoteSafe(aa))
argStr = simenv.OptionsManager.BuildOptionString(stripArguments)
command = "%s %s %s" % (command, cmdargs, argStr)
# lets append --machine <remotemachinename> to the arg string to avoid any confusion.
command = "%s --machine=%s" % (command, self.machineName)
cmd = simlib.GetSSHCommand(simenv.LocalMachine, self.machineName,
command, sshargs, self.RemotePath)
machineEntry = simenv.ConfigurationDatabase.GetMachine(
self.machineName)
localSourceBaseDir = simlib.GetLocalSourceBaseDir()
source_name = simlib.GetDirSuffix(localSourceBaseDir)
local_sourcedir = simlib.BuildPath(localSourceBaseDir, source_name)
DefineDatabase.Set('SOURCEDIR', local_sourcedir)
cmd = DefineDatabase.SubAll(cmd)
DefineDatabase.Set('USER', machineEntry.user)
localsshsetup = DefineDatabase.SubAll(machineEntry.localsshsetup)
cmd = "{ %s; } && { %s; } && %s" % (self.localEnvSetup, localsshsetup,
cmd)
#dprint("Executing Command: %s" % cmd)
ret = simlib.ExecuteCommand(cmd)
(status, signal) = (ret >> 8, ret & 0xff)
if status == 0xff:
fatal("Could not execute command \"%s\"" % cmd)
def GetExecHost(restart):
DefineDatabase = simsubs.DefineDatabase()
(machine, machineEntry, sourceBaseDir) = simlib.GetLocalEnvironment()
job_id = restart.GetJobId()
job_status = 'U'
if job_id != -1:
job_status = GetJobStatus(job_id)
if job_status != 'R':
warning("Job is not running, cannot retreive exechost")
return None
simlib.VerifyKeys(machineEntry, ['exechost', 'exechostpattern'])
DefineDatabase.Set('JOB_ID', job_id)
DefineDatabase.Set('USER', machineEntry.user)
DefineDatabase.Set('SIMULATION_NAME', restart.SimulationName)
exechost = DefineDatabase.SubAll(machineEntry.GetKey('exechost'))
exechostpattern = DefineDatabase.SubAll(
machineEntry.GetKey('exechostpattern'))
output, rc = simlib.ExecuteCommand(exechost, True)
rx = re.compile(exechostpattern, re.MULTILINE)
matches = rx.search(output)
if rc or matches == None:
warning("Unable to retrieve exechost using pattern %s" %
exechostpattern)
return None
return matches.group(1)
def GetBaseDir(machineEntry):
DefineDatabase = simsubs.DefineDatabase()
if simenv.OptionsManager.HasOption('basedir'):
basedir = simenv.OptionsManager.GetOption('basedir')
else:
machineEntry = simenv.LocalMachineEntry
VerifyKeys(machineEntry, ['user', 'basedir'])
user = machineEntry.user
DefineDatabase.Set('USER', user)
basedir = DefineDatabase.SubAll(machineEntry.basedir)
if not(basedir[0] == "/"):
cwd = os.getcwd()
basedir = "%s%s%s" % (cwd, os.sep, basedir)
#if not(os.path.exists(basedir)):
# dprint('Cannot access source base directory "%s"' % basedir)
return basedir
def __init__(self, tree, parentTree=None):
self.IniTree = tree
self.DefineDatabase = simsubs.DefineDatabase()
self.ParentTree = parentTree
def GetJobStatus(job_id):
# TODO: there needs to be documentation stating what the possible
# return values are
DefineDatabase = simsubs.DefineDatabase()
(machine, machineEntry, sourceBaseDir) = simlib.GetLocalEnvironment()
simlib.VerifyKeys(machineEntry, [
'getstatus', 'queuedpattern', 'runningpattern', 'statuspattern', 'user'
])
status_command = machineEntry.GetKey('getstatus')
status_pattern = machineEntry.GetKey('statuspattern')
queued_pattern = machineEntry.GetKey('queuedpattern')
running_pattern = machineEntry.GetKey('runningpattern')
holding_pattern = machineEntry.GetKey('holdingpattern')
user = machineEntry.GetKey('user')
DefineDatabase.Set('USER', user)
DefineDatabase.Set('JOB_ID', job_id)
status_command = DefineDatabase.SubAll(status_command)
status_pattern = DefineDatabase.SubAll(status_pattern)
queued_pattern = DefineDatabase.SubAll(queued_pattern)
running_pattern = DefineDatabase.SubAll(running_pattern)
holding_pattern = DefineDatabase.SubAll(holding_pattern)
#capture output.
output, ret = simlib.ExecuteCommand(status_command, output=True)
# U == unknown?
status = 'U'
if ret != None:
lines = output.split("\n")
#first, lets see if the job is still in the queue, regardless of whether or not we can determine the job status
# TODO: counting lines doesn't help detecting errors
InQueue = lines.count(job_id) > 0
matched = []
for line in lines:
matches = re.search(status_pattern, line)
if matches != None:
# queued_pattern
if re.search(queued_pattern, line):
status = 'Q'
matched.append(queued_pattern)
# running_pattern
if re.search(running_pattern, line):
status = 'R'
matched.append(running_pattern)
# holding_pattern
if holding_pattern != None:
if re.search(holding_pattern, line):
status = 'H'
matched.append(holding_pattern)
# TODO: matches is set in the last loop iteration only; this if
# statement is bogus
if matches > 1:
# TODO: output a better error message; the list of patterns
# alone is not useful because patterns are difficult to read;
# better would be also what state the patterns are for
fatal("multiple status patterns matched: %s" % matched)
if InQueue and status == 'U':
status = 'E'
if status == 'U':
warning("job status is U")
dprint("queue status return code is: %d" % ret)
dprint("queue status output is:")
for line in lines:
dprint(" lines=[%s]" % line)
dprint("patterns are:")
dprint(" status_pattern=[%s]" % status_pattern)
dprint(" queued_pattern=[%s]" % queued_pattern)
dprint(" running_pattern=[%s]" % running_pattern)
dprint(" holding_pattern=[%s]" % holding_pattern)
return status
def PrepConfiguration(configName):
DefineDatabase = simsubs.DefineDatabase()
machineEntry = simenv.LocalMachineEntry
req_keys = ['user', 'email', 'make']
simlib.VerifyKeys(machineEntry, req_keys)
user = machineEntry.GetKey("user")
email = machineEntry.GetKey("email")
DefineDatabase.Set('USER', user)
DefineDatabase.Set('EMAIL', email)
make = DefineDatabase.SubAll(machineEntry.GetKey('make'))
subfolders = ['bindings', 'build', 'config-data', 'lib', 'scratch']
configBase = simlib.BuildPath(simenv.CONFIGS_PATH, configName)
propertiesFile = simlib.BuildPath(configBase, "properties.ini")
if not os.path.exists(configBase):
try:
os.mkdir(configBase)
except OSError:
pass
#simlib.ExecuteCommand("mkdir %s" % configBase)
for sub in subfolders:
sf = simlib.BuildPath(configBase, sub)
simlib.ExecuteCommand("mkdir %s" % sf)
else: # The config dir exists already
if not os.path.exists(propertiesFile):
fatal(
"Configuration %s has no properties.ini file. Either this is not a SimFactory configuration, or the configuration has become corrupted. If you think the configuration has been corrupted, you will need to delete it and run the sim build command again."
% (configName))
prop = simproperties.SimProperties()
prop.init(propertiesFile)
build_reconfig = GetConfigValue(prop, "reconfig")
build_clean = GetConfigValue(prop, "clean")
build_debug = GetConfigValue(prop, "debug")
build_optimise = GetConfigValue(prop, "optimise")
build_unsafe = GetConfigValue(prop, "unsafe")
build_profile = GetConfigValue(prop, "profile")
if build_debug:
display("Disabling optimisation because debug was selected")
build_optimise = False
# update our properties.ini
prop.RemoveProperty("reconfig")
prop.RemoveProperty("clean")
prop.AddProperty("debug", build_debug)
prop.AddProperty("optimise", build_optimise)
prop.AddProperty("unsafe", build_unsafe)
prop.AddProperty("profile", build_profile)
# Remove the executable, so that it is not accidentally used
# while it is rebuilt. Do this before the configuration is
# modified in any way, but only after some basic error
# checking.
try:
os.remove("exe/cactus_%s" % configName)
except OSError:
pass
try:
shutil.rmtree("exe/%s" % configName, ignore_errors=True)
except OSError:
pass
prop.Save()
storedOptions = simlib.BuildPath(configBase, "OptionList")
hasStoredOptions = os.path.exists(storedOptions)
cctkConfigPath = simlib.BuildPath(configBase, "config-data",
"cctk_Config.h")
hasCompleteConfig = os.path.exists(cctkConfigPath)
if hasStoredOptions:
storedOptionSettings = simlib.GetFileContents(storedOptions)
else:
storedOptionSettings = None
info("HasStoredOptions: %s" % hasStoredOptions)
removeConfig = False # default behaviour
# we are only updating the OptionList if its explicitly specified with --optionlist, really.
if hasStoredOptions and not simenv.OptionsManager.RawOptionDefined(
"optionlist"):
info("Use stored options list: %s" % storedOptions)
optionlist = storedOptions
else:
optionlist = simlib.GetOptionList(False)
info("optionlist is: %s" % optionlist)
if optionlist == None or not os.path.exists(optionlist):
display("Warning: no option list specified, using blank option list")
optionSettings = str()
else:
optionSettings = simlib.GetFileContents(optionlist)
DefineDatabase.AddReplacement("DEBUG", simlib.BoolToYesNo(build_debug))
DefineDatabase.AddReplacement("OPTIMISE",
simlib.BoolToYesNo(build_optimise))
DefineDatabase.AddReplacement("UNSAFE", simlib.BoolToYesNo(build_unsafe))
DefineDatabase.AddReplacement("PROFILE", simlib.BoolToYesNo(build_profile))
# add support for CACHELINE_BYTES and CACHE_SIZE
CACHELINE_BYTES = simenv.LocalMachineEntry.GetKey("L3linesize")
if CACHELINE_BYTES != None:
DefineDatabase.AddReplacement("CACHELINE_BYTES", CACHELINE_BYTES)
CACHE_SIZE = simenv.LocalMachineEntry.GetKey("L3size")
if CACHE_SIZE != None:
DefineDatabase.AddReplacement("CACHE_SIZE", CACHE_SIZE)
optionSettings = DefineDatabase.SubAll(optionSettings)
hasOutdatedConfig = (not hasCompleteConfig) or (
storedOptionSettings
== None) or (storedOptionSettings != optionSettings)
if hasOutdatedConfig:
info("hasCompleteConfig: %s" % hasCompleteConfig)
info("hasOutdatedConfig: %s" % hasOutdatedConfig)
info("build_reconfig: %s" % build_reconfig)
# if build virtual, disable configging.
if hasOutdatedConfig or build_reconfig:
if storedOptionSettings != None:
oldVersion = simlib.GetVersion(storedOptionSettings)
else:
oldVersion = ""
newVersion = simlib.GetVersion(optionSettings)
info("oldVersion %s, newVersion %s" % (oldVersion, newVersion))
removeConfig = removeConfig or newVersion != oldVersion
display("Reconfiguring %s" % configName)
simlib.RemoveFile("%s.old" % storedOptions)
simlib.RenameFile(storedOptions, "%s.old" % storedOptions)
# TODO: Write new option list only after the make *-realclean below!
display("Writing configuration to: %s" % storedOptions)
simlib.WriteContents(storedOptions, optionSettings)
#"echo yes | { $make $configuration_name-config options=configs/$configuration_name/OptionList; }";
if not simenv.OptionsManager.GetOption('virtual'):
cmd = "cd %s && echo yes | { %s %s-config options=%s; } 2>&1" % (
simenv.CACTUS_PATH, make, configName, storedOptions)
ret = simlib.ExecuteCommand(cmd)
if ret != 0:
sys.exit(1)
#force a rebuild
simlib.RemoveFile(
simlib.BuildPath(configBase, 'config-data', 'make.thornlist'))
#remove the old config if necessary
if removeConfig:
display("Complete rebuild required")
else:
if not hasStoredOptions:
fatal(
"Configuration %s has no option list, and no option list was specified."
% configName)
if not simenv.OptionsManager.GetOption('virtual'):
info("build_clean: %s" % build_clean)
info("removeConfig: %s" % removeConfig)
if build_clean or removeConfig:
display("Cleaning %s" % configName)
ret = simlib.ExecuteCommand("cd %s && %s %s-realclean 2>&1" %
(simenv.CACTUS_PATH, make, configName))
if ret != 0:
sys.exit(1)
## deal with submit script now
storedSubmitScript = simlib.BuildPath(configBase, "SubmitScript")
hasStoredSubmitScript = os.path.exists(storedSubmitScript)
if simenv.OptionsManager.GetOption('no-submitscript'):
if hasStoredSubmitScript:
display("Removing stored submit script for configuration %s" %
configName)
os.unlink(storedSubmitScript)
warning("empty submit script will disable submission")
else:
# the submit script is entirely optional. no submit script
# just means submission is disabled.
submitscript = simlib.GetSubmitScript()
info("SubmitScript is: %s" % submitscript)
if submitscript != None:
sContents = simlib.GetFileContents(submitscript)
ssContents = simlib.GetFileContents(storedSubmitScript)
submitScriptOutdated = not hasStoredSubmitScript or simenv.OptionsManager.RawOptionDefined(
"submitscript")
if sContents != ssContents:
warning("default submit script contents have changed")
if (submitScriptOutdated or build_reconfig):
display("Updated script file for configuration %s" %
configName)
simlib.RemoveFile("%s.old" % storedSubmitScript)
simlib.RenameFile(storedSubmitScript,
"%s.old" % storedSubmitScript)
shutil.copy(submitscript, storedSubmitScript)
else:
if not hasStoredSubmitScript:
warning("empty submit script will disable submission")
## deal with run script now
storedRunScript = simlib.BuildPath(configBase, "RunScript")
hasStoredRunScript = os.path.exists(storedRunScript)
runscript = simlib.GetRunScript(False)
info("RunScript is: %s" % runscript)
if runscript != None:
sContents = simlib.GetFileContents(runscript)
ssContents = simlib.GetFileContents(storedRunScript)
runScriptOutdated = not hasStoredRunScript or simenv.OptionsManager.RawOptionDefined(
"runscript")
if sContents != ssContents:
warning("default run script contents have changed")
if (runScriptOutdated or build_reconfig):
display("Updated runscript file for configuration %s" % configName)
simlib.RemoveFile("%s.old" % storedRunScript)
simlib.RenameFile(storedRunScript, "%s.old" % storedRunScript)
shutil.copy(runscript, storedRunScript)
else:
if not hasStoredRunScript:
fatal(
"Configuration %s has no run script, and no run script file was specified"
% configName)
## deal with thorn list now
storedThornList = simlib.BuildPath(configBase, "ThornList")
hasStoredThornList = os.path.exists(storedThornList)
needThornList = not hasStoredThornList
thornlist = simlib.GetThornList(needThornList)
info("ThornList is: %s" % thornlist)
if thornlist != None:
tContents = simlib.GetThornListContents(thornlist)
ttContents = simlib.GetFileContents(storedThornList)
thornListOutdated = not hasStoredThornList or simenv.OptionsManager.RawOptionDefined(
"thornlist")
if tContents != ttContents:
warning("default thorn list contents have changed")
if (thornListOutdated or build_reconfig):
display("Updated thorn list for configuration %s" % configName)
simlib.RemoveFile("%s.old" % storedThornList)
simlib.RenameFile(storedThornList, "%s.old" % storedThornList)
simlib.WriteContents(storedThornList, tContents)
else:
if not hasStoredThornList:
fatal(
"Configuration %s has no thorn list, and no thorn list file was specified"
% configName)
def CompileCommand(machineName, rsyncInfo, paths, oldRsync=False):
global localMachineName
global localMachineEntry
global local_sourcebasedir
global local_suffix
DefineDatabase = simsubs.DefineDatabase()
(rsynccmd, rsyncopts) = rsyncInfo
rsyncversion = simlib.RsyncVersion(rsyncInfo)
if machineName == localMachineName:
fatal("cannot sync to local machine")
machineEntry = simenv.ConfigurationDatabase.GetMachine(machineName)
# get our IO machine -- if iomachine doesn't exist, it returns itself.
ioMachine = simlib.GetIOMachine(machineName)
if ioMachine != machineName:
machineEntry = simenv.ConfigurationDatabase.GetMachine(ioMachine)
machineName = ioMachine
# make sure we have all the info we need.
mreq_keys = [
'user', 'sourcebasedir', 'hostname', 'rsynccmd', 'rsyncopts', 'sshcmd',
'sshopts'
]
simlib.VerifyKeys(machineEntry, mreq_keys)
DefineDatabase.Set('USER', machineEntry.user)
sourcebasedir = DefineDatabase.SubAll(machineEntry.sourcebasedir)
# There need to be two define databases, one for the local and
# another for the remote system. Currently, USER is for the
# remote and SOURCEDIR is for the local system -- this is
# inconsistent.
#local_sourcebasedir = DefineDatabase.SubAll(localMachineEntry.sourcebasedir)
source_name = local_suffix
#source_name = simlib.GetDirSuffix(local_sourcebasedir)
local_sourcedir = simlib.BuildPath(local_sourcebasedir, source_name)
DefineDatabase.Set('SOURCEDIR', local_sourcedir)
localsshsetup = DefineDatabase.SubAll(machineEntry.localsshsetup)
# sshcmd = machineEntry.sshcmd
# sshopts = machineEntry.sshopts
# sshcmd = "%s %s" % (sshcmd, sshopts)
# trampoline = simlib.GetTrampoline(ioMachine)
# if trampoline:
# tentry = simenv.ConfigurationDatabase.GetMachine(trampoline)
# trampoline_sourcebasedir = simlib.GetSourceBaseDir(tentry)
# trampoline_sourcedir = os.path.join(trampoline_sourcebasedir, source_name)
# DefineDatabase.Set('SOURCEDIR', trampoline_sourcedir)
# sshcmd = DefineDatabase.SubAll(sshcmd)
# DefineDatabase.Set('SOURCEDIR', local_sourcedir)
# sshcmd = simlib.GetSSHCommand(trampoline, sshcmd)
# sshcmd = DefineDatabase.SubAll(sshcmd)
sshcmd = simlib.GetSSHCommand(localMachineName, ioMachine, None)
# If there is more than one explicit path given only accept
# top-level directories This could be extended to do the right
# thing when multiple explicit paths are given but would require
# running rsync multiple times
rsyncfiles = []
if len(paths) == 1:
rsyncfiles = paths
[head, tail] = os.path.split(rsyncfiles[0])
local_suffix = os.path.join(local_suffix, head)
else:
for file in paths:
if file == '':
continue
[head, tail] = os.path.split(file)
if head != "":
fatal(
"Only top level paths may be specified when syncing multiple paths but '%s' given."
% file)
elif not os.path.exists(tail):
warning("Specified sync path '%s' not found." % tail)
else:
rsyncfiles.append(tail)
# Note:
# - We omit --times, since the target system should recompile when
# a file changed there
# - We omit --group and --owner, since this probably works for
# root only, and all files on the targe system should rather
# belong to the user
# - We omit -D, since we don't expect to have devices or other
# special files
# - Since we omit --times, we add --checksum so that differing
# modification times don't lead to a retransmission
# - NOTE: rsync before 3.0.8 has an error that makes --checksum
# unreliable under certain circumstances (see
# <https://rsync.samba.org/ftp/rsync/src/rsync-3.0.8-NEWS>). We
# cannot use --ignore-times instead because this modifies all
# files' timestamps (and is also very expensive).
rsyncoptions = [
'--checksum',
'--compress',
'--delete',
'--hard-links',
'--links',
'--partial',
'--perms',
'--progress',
'--recursive',
'--sparse',
'--stats',
#'--times',
'--verbose'
]
# We need to use a different rules file for pre 3.0.0 versions of rsync
ruleFile = 'filter.rules'
if rsyncversion[0] < 3 or oldRsync:
info(
'Old rsync version in use (local version is %s.%s.%s). Upgrade to at least 3.0.0 both locally and remotely for best performance.'
% rsyncversion)
ruleFile = 'filter.prersync3.rules'
userRuleFile = "%s/etc/%s" % (simenv.BASE_PATH, "filter.local.rules")
if os.path.exists(userRuleFile):
rsyncoptions.append("--filter 'merge %s'" % (userRuleFile))
rsyncoptions.append("--filter 'merge %s/etc/%s'" %
(simenv.BASE_PATH, ruleFile))
fullpath = "%s@%s:%s%s%s" % (machineEntry.user, machineEntry.hostname,
sourcebasedir, os.sep, local_suffix)
arguments = " ".join(simlib.GetArguments())
cmd = "%s --rsh=%s --rsync-path=%s %s %s %s" % (
rsynccmd, simlib.QuoteSafe(sshcmd),
simlib.QuoteSafe(machineEntry.rsynccmd), rsyncopts,
machineEntry.rsyncopts, arguments)
cmd = "%s %s" % (cmd, " ".join(rsyncoptions))
cmd = "%s %s" % (cmd, " ".join(rsyncfiles))
cmd = "%s %s" % (cmd, fullpath)
mkdirpath = os.path.join(sourcebasedir, local_suffix)
mkdircmd = "mkdir -p %s" % simlib.QuoteSafe(mkdirpath)
#mkdircmd = "%s %s@%s %s" % (sshcmd, machineEntry.user, machineEntry.hostname, simlib.QuoteSafe(mkdircmd))
mkdircmd = simlib.GetSSHCommand(localMachineName, ioMachine, mkdircmd)
cmd = "cd %s && { %s; } && { %s || { %s && %s; } }" % (
simlib.QuoteSafe(local_sourcedir), localsshsetup, cmd, mkdircmd, cmd)
return cmd