def load(self, name): """Loads all values from configuration and command line overrides into data structures suitable for use by the TemplateWriter object. """ resolvedName = envString.resolve(name) configuration = CondorConfig() configuration.load(resolvedName) self.defaults = {} if configuration.platform.nodeSetRequired and self.opts.nodeSet is None: print("error: nodeset parameter required by this platform") sys.exit(10) tempDefaultRoot = Template(configuration.platform.defaultRoot) self.defaults["DEFAULT_ROOT"] = tempDefaultRoot.substitute( USER_NAME=self.commandLineDefaults["USER_NAME"]) tempLocalScratch = Template(configuration.platform.localScratch) self.defaults["LOCAL_SCRATCH"] = \ tempLocalScratch.substitute(USER_NAME=self.commandLineDefaults["USER_NAME"]) self.defaults["IDS_PER_JOB"] = configuration.platform.idsPerJob self.defaults["DATA_DIRECTORY"] = envString.resolve(configuration.platform.dataDirectory) self.defaults["FILE_SYSTEM_DOMAIN"] = configuration.platform.fileSystemDomain self.defaults["EUPS_PATH"] = configuration.platform.eupsPath self.defaults["MANAGER_SOFTWARE_HOME"] = configuration.platform.manager_software_home platform_dir = lsst.utils.getPackageDir("ctrl_platform_"+self.opts.platform) self.defaults["PLATFORM_DIR"] = platform_dir self.manager = configuration.platform.manager self.setup_using = configuration.platform.setup_using
def __init__(self, platform): # can handle both grid-proxy and ssh logins self.remoteLoginCmd = "/usr/bin/gsissh" # can handle both grid-proxy and ssh copies self.remoteCopyCmd = "/usr/bin/gsiscp" configFileName = "$HOME/.lsst/condor-info.py" fileName = envString.resolve(configFileName) condorInfoConfig = CondorInfoConfig() condorInfoConfig.load(fileName) platformPkgDir = lsst.utils.getPackageDir("ctrl_platform_" + platform) configName = os.path.join(platformPkgDir, "etc", "config", "pbsConfig.py") allocationConfig = AllocationConfig() allocationConfig.load(configName) self.userName = condorInfoConfig.platform[platform].user.name self.hostName = allocationConfig.platform.loginHostName self.utilityPath = allocationConfig.platform.utilityPath
def createConfiguration(self, input): """ creates a new Orca configuration file @param input: template to use for value substitution @return the newly created Orca configuration file """ resolvedInputName = envString.resolve(input) if self.opts.verbose: print("creating configuration using ", resolvedInputName) template = TemplateWriter() substitutes = self.defaults.copy() for key in self.commandLineDefaults: val = self.commandLineDefaults[key] if val is not None: substitutes[key] = self.commandLineDefaults[key] substitutes["CTRL_EXECUTE_SETUP_PACKAGES"] = self.getSetupPackages() configDir = os.path.join(substitutes["LOCAL_SCRATCH"], "configs") if not os.path.exists(configDir): os.mkdir(configDir) self.outputFileName = os.path.join(configDir, "%s.config" % (self.runid)) if self.opts.verbose: print("writing new configuration to ", self.outputFileName) template.rewrite(resolvedInputName, self.outputFileName, substitutes) return self.outputFileName
def main(): """Allocates Condor glide-in nodes a scheduler on a remote Node. """ p = AllocatorParser(sys.argv[0]) platform = p.getPlatform() # load the CondorConfig file platformPkgDir = lsst.utils.getPackageDir("ctrl_platform_" + platform) execConfigName = os.path.join(platformPkgDir, "etc", "config", "execConfig.py") resolvedName = envString.resolve(execConfigName) configuration = CondorConfig() configuration.load(resolvedName) # create the plugin class schedulerName = configuration.platform.scheduler schedulerClass = NamedClassFactory.createClass("lsst.ctrl.execute." + schedulerName + "Plugin") # create the plugin scheduler = schedulerClass(platform, p.getArgs(), configuration, "$HOME/.lsst/condor-info.py") # submit the request scheduler.submit(platform, platformPkgDir)
def loadAllocationConfig(self, name, suffix): """Loads all values from allocationConfig and command line overrides into data structures suitable for use by the TemplateWriter object. """ resolvedName = envString.resolve(name) allocationConfig = AllocationConfig() if not os.path.exists(resolvedName): raise RuntimeError("%s was not found." % resolvedName) allocationConfig.load(resolvedName) self.defaults["QUEUE"] = allocationConfig.platform.queue self.defaults["EMAIL_NOTIFICATION"] = allocationConfig.platform.email self.defaults["HOST_NAME"] = allocationConfig.platform.loginHostName self.defaults["UTILITY_PATH"] = allocationConfig.platform.utilityPath if self.opts.glideinShutdown is None: self.defaults["GLIDEIN_SHUTDOWN"] = str(allocationConfig.platform.glideinShutdown) else: self.defaults["GLIDEIN_SHUTDOWN"] = str(self.opts.glideinShutdown) if self.opts.nodeSet is None: self.defaults["NODE_SET"] = self.createNodeSetName() else: self.defaults["NODE_SET"] = self.opts.nodeSet nodeSetName = self.defaults["NODE_SET"] if self.opts.outputLog is not None: self.defaults["OUTPUT_LOG"] = self.opts.outputLog else: self.defaults["OUTPUT_LOG"] = "%s.out" % nodeSetName if self.opts.errorLog is not None: self.defaults["ERROR_LOG"] = self.opts.errorLog else: self.defaults["ERROR_LOG"] = "%s.err" % nodeSetName # This is the TOTAL number of cores in the job, not just the total # of the cores you intend to use. In other words, the total available # on a machine, times the number of machines. totalCoresPerNode = allocationConfig.platform.totalCoresPerNode self.commandLineDefaults["TOTAL_CORE_COUNT"] = self.opts.nodeCount * totalCoresPerNode self.uniqueIdentifier = self.createUniqueIdentifier() # write these pbs and config files to {LOCAL_DIR}/configs self.configDir = os.path.join(self.defaults["LOCAL_SCRATCH"], "configs") if not os.path.exists(self.configDir): os.makedirs(self.configDir) self.submitFileName = os.path.join(self.configDir, "alloc_%s.%s" % (self.uniqueIdentifier, suffix)) self.condorConfigFileName = os.path.join(self.configDir, "condor_%s.config" % self.uniqueIdentifier) self.defaults["GENERATED_CONFIG"] = os.path.basename(self.condorConfigFileName) self.defaults["CONFIGURATION_ID"] = self.uniqueIdentifier return allocationConfig
def __init__(self, platform, opts, configuration, condorInfoFileName): """Constructor @param platform: target platform for PBS submission @param opts: options to override """ self.opts = opts self.defaults = {} self.configuration = configuration fileName = envString.resolve(condorInfoFileName) condorInfoConfig = CondorInfoConfig() condorInfoConfig.load(fileName) self.platform = platform # Look up the user's name and home directory in the # $HOME/.lsst/condor-info.py file # If the platform is lsst, and the user_name or user_home # is not in there, then default to user running this # command and the value of $HOME, respectively. user_name = None user_home = None for name in condorInfoConfig.platform: if name == self.platform: user_name = condorInfoConfig.platform[name].user.name user_home = condorInfoConfig.platform[name].user.home if self.platform == "lsst": if user_name is None: user_name = pwd.getpwuid(os.geteuid()).pw_name if user_home is None: user_home = os.getenv('HOME') if user_name is None: raise RuntimeError( "error: %s does not specify user name for platform == %s" % (condorInfoFileName, self.platform)) if user_home is None: raise RuntimeError( "error: %s does not specify user home for platform == %s" % (condorInfoFileName, self.platform)) self.defaults["USER_NAME"] = user_name self.defaults["USER_HOME"] = user_home self.commandLineDefaults = {} self.commandLineDefaults["NODE_COUNT"] = self.opts.nodeCount self.commandLineDefaults["SLOTS"] = self.opts.slots self.commandLineDefaults["WALL_CLOCK"] = self.opts.maximumWallClock self.commandLineDefaults["QUEUE"] = self.opts.queue if self.opts.email == "no": self.commandLineDefaults["EMAIL_NOTIFICATION"] = "#" self.load()
def __init__(self, platform, opts, configuration, condorInfoFileName): """Constructor @param platform: target platform for PBS submission @param opts: options to override """ self.opts = opts self.defaults = {} self.configuration = configuration fileName = envString.resolve(condorInfoFileName) condorInfoConfig = CondorInfoConfig() condorInfoConfig.load(fileName) self.platform = platform # Look up the user's name and home directory in the # $HOME/.lsst/condor-info.py file # If the platform is lsst, and the user_name or user_home # is not in there, then default to user running this # command and the value of $HOME, respectively. user_name = None user_home = None for name in condorInfoConfig.platform: if name == self.platform: user_name = condorInfoConfig.platform[name].user.name user_home = condorInfoConfig.platform[name].user.home if self.platform == "lsst": if user_name is None: user_name = pwd.getpwuid(os.geteuid()).pw_name if user_home is None: user_home = os.getenv('HOME') if user_name is None: raise RuntimeError("error: %s does not specify user name for platform == %s" % (condorInfoFileName, self.platform)) if user_home is None: raise RuntimeError("error: %s does not specify user home for platform == %s" % (condorInfoFileName, self.platform)) self.defaults["USER_NAME"] = user_name self.defaults["USER_HOME"] = user_home self.commandLineDefaults = {} self.commandLineDefaults["NODE_COUNT"] = self.opts.nodeCount self.commandLineDefaults["CPUS"] = self.opts.cpus self.commandLineDefaults["WALL_CLOCK"] = self.opts.maximumWallClock self.commandLineDefaults["QUEUE"] = self.opts.queue if self.opts.email == "no": self.commandLineDefaults["EMAIL_NOTIFICATION"] = "#" self.load()
def createFile(self, input, output): """Creates a new file, using "input" as a Template, and writes the new file to output. Returns ------- outfile : `str` The newly created file name """ resolvedInputName = envString.resolve(input) if self.opts.verbose: print("creating file using %s" % resolvedInputName) template = TemplateWriter() # Uses the associative arrays of "defaults" and "commandLineDefaults" # to write out the new file from the template. # The commandLineDefaults override values in "defaults" substitutes = self.defaults.copy() for key in self.commandLineDefaults: val = self.commandLineDefaults[key] if val is not None: substitutes[key] = self.commandLineDefaults[key] template.rewrite(resolvedInputName, output, substitutes) return output
def main(): """Allocates Condor glide-in nodes a scheduler on a remote Node. """ p = AllocatorParser(sys.argv[0]) platform = p.getPlatform() # load the CondorConfig file platformPkgDir = lsst.utils.getPackageDir("ctrl_platform_"+platform) execConfigName = os.path.join(platformPkgDir, "etc", "config", "execConfig.py") resolvedName = envString.resolve(execConfigName) configuration = CondorConfig() configuration.load(resolvedName) # create the plugin class schedulerName = configuration.platform.scheduler schedulerClass = NamedClassFactory.createClass("lsst.ctrl.execute." + schedulerName + "Plugin") # create the plugin scheduler = schedulerClass(platform, p.getArgs(), configuration, "$HOME/.lsst/condor-info.py") # submit the request scheduler.submit(platform, platformPkgDir)
def __init__(self, seqFileName): """Constructor @param seqFileName file name to operate on """ self.fileName = envString.resolve(seqFileName)
class UserInfoConfig(pexConfig.Config): """ User information """ name = pexConfig.Field(doc="user login name", dtype=str, default=None) home = pexConfig.Field(doc="user home directory", dtype=str, default=None) class UserConfig(pexConfig.Config): """ User specific information """ user = pexConfig.ConfigField(doc="user", dtype=UserInfoConfig) class CondorInfoConfig(pexConfig.Config): """A pex_config file describing the platform specific information about remote user logins. """ platform = pexConfig.ConfigChoiceField("platform info", FakeTypeMap(UserConfig)) if __name__ == "__main__": config = CondorInfoConfig() filename = "$HOME/.lsst/condor-info.py" filename = envString.resolve(filename) config.load(filename) for i in config.platform: print(i)
def __init__(self, opts, configFileName): """Constructor @param opts: options to override """ self.opts = opts self.setup_using = None self.manager = None self.defaults = {} fileName = envString.resolve(configFileName) condorInfoConfig = CondorInfoConfig() condorInfoConfig.load(fileName) self.platform = self.opts.platform # Look up the user's name and home directory in the # $HOME//.lsst/condor-info.py file. If the platform # is lsst, and the user_name or user_home is not in # there, then default to user running this comman # and the value of $HOME, respectively. user_name = None user_home = None for name in list(condorInfoConfig.platform.keys()): if name == self.platform: user_name = condorInfoConfig.platform[name].user.name user_home = condorInfoConfig.platform[name].user.home # If we're on the lsst platform and the condorInfoConfig didn't # have an entry for lsst user name and home, set to reasonable values # These really do need to be set for all the other platforms, since # while the user name may be the same, it's unlikely the home # directory will be. if self.platform == "lsst": if user_name is None: user_name = pwd.getpwuid(os.geteuid()).pw_name if user_home is None: user_home = os.getenv('HOME') if user_name is None: raise RuntimeError("error: %s does not specify user name for platform %s" % (configFileName, self.platform)) if user_home is None: raise RuntimeError("error: %s does not specify user home for platform %s" % (configFileName, self.platform)) self.commandLineDefaults = {} self.commandLineDefaults["USER_NAME"] = user_name self.commandLineDefaults["USER_HOME"] = user_home self.commandLineDefaults["DEFAULT_ROOT"] = self.opts.defaultRoot self.commandLineDefaults["LOCAL_SCRATCH"] = self.opts.localScratch self.commandLineDefaults["DATA_DIRECTORY"] = self.opts.dataDirectory self.commandLineDefaults["IDS_PER_JOB"] = self.opts.idsPerJob if self.opts.nodeSet is None: self.commandLineDefaults["NODE_SET"] = "" else: self.commandLineDefaults["NODE_SET"] = self.opts.nodeSet if self.opts.inputDataFile is None: self.commandLineDefaults["INPUT_DATA_FILE"] = None else: self.commandLineDefaults["INPUT_DATA_FILE"] = os.path.abspath(self.opts.inputDataFile) self.commandLineDefaults["FILE_SYSTEM_DOMAIN"] = self.opts.fileSystemDomain self.commandLineDefaults["EUPS_PATH"] = self.opts.eupsPath # override user name, if given if self.opts.user_name is not None: self.commandLineDefaults["USER_NAME"] = self.opts.user_name # override user home, if given if self.opts.user_home is not None: self.commandLineDefaults["USER_HOME"] = self.opts.user_home if self.opts.runid is not None: self.runid = self.opts.runid else: self.runid = self.createRunId() if self.opts.dagscript is not None: self.commandLineDefaults["DAGSCRIPT"] = self.opts.dagscript if self.opts.inputscript is not None: self.commandLineDefaults["INPUTSCRIPT"] = self.opts.inputscript if self.opts.platformConfig is not None: self.commandLineDefaults["PLATFORM_CONFIG"] = self.opts.platformConfig self.commandLineDefaults["COMMAND"] = self.opts.command if self.commandLineDefaults["INPUT_DATA_FILE"] is not None: self.commandLineDefaults["COMMAND"] = self.commandLineDefaults["COMMAND"]+" ${id_option}"