def __init__(self,input): self.bintype = input['Architecture'] self.comment = input['Comments'] self.darname = input['DARFileName'] self.status = input['DARStatus' ] self.release = input['Version' ] self.project = input['Application'] self.idNum = input['DARFileID' ] # URLs of files to be added to the users CWD: self.extra_url = [] # Getting url of the external xml file from the comment: xml_url_pattern = re.compile('.*(http://.*\.xml).*') res = xml_url_pattern.match(self.comment) if res: self.extra_url.append(res.group(1)) # Support for both scram and scramv1: # First get scram version for this release: majorV = getScramVersion(getReleaseTop(self.project, self.release))[:2] if majorV == "V0": self.scram = SCRAM('scram') elif majorV == "V1": self.scram = SCRAM('scramv1') else: sys.exit("ERROR! could not define scram executable for" + \ "scram version " + majorV) # Get current architecture. # Check to be added here: # - scram will create project area even if release is not available # for this platform # - architecture should comply with the current system # Get current architecture from scram infoOut( 'scram architecture is '+ self.scram.getScramArch())
def __init__(self, line): Lister.__init__(self) self.line = line self.name = '' self.args = None infoOut ('processing command: '+ line) self.processCommand()
def saveLog(self): lf=open(self.sessionStdoutLog,'w') lf.writelines(self.s.allOut) lf.close() lf=open(self.sessionHistoryLog,'w') lf.writelines(self.s.commands) lf.close() infoOut("Session logfiles can be found in the directory "+self.logdir)
def saveMetadata(self, metadataFile): """ Metadata.saveMetadata Saves metadata info using the pickle module """ # Save to file: # outputFile = open (file, 'w') infoOut('Saving dar metadata in file: ' + metadataFile) # This saves human readable user info: #f.write(self.userInfo()) # This writes object's listing provided by class Lister: # f.write(str(self)) # This saves the instance in python internal object stream format pickle.dump(self, open(metadataFile, 'w'))
def __init__(self, tmpdir, program): """ Makes necessary checks and sets RefDBDAR attributes""" #check that CMS environment is set: if not os.environ.has_key('CMS_PATH'): message = """CMS_PATH variable is not defined. Please set CMS environment""" sys.exit(message) self.tmpdir = tmpdir infoOut( 'temporary directory is '+self.tmpdir) if notWritable(self.tmpdir): sys.exit(notWritable(self.tmpdir)) # Collect a list of warnings: self.allWarnings = [] # Start session to execute shell commands self.s=Session() # Create every time new build directory in tmpdir pool # using datestamp : self.blddir = self.tmpdir + '/' + str(time.time()) self.s.run('mkdir -p ' + self.blddir) # Define log files (all in one directory): self.logdir = self.blddir + '/logfiles/' self.s.run('mkdir -p ' + self.logdir) self.timingLog = Logger(self.logdir+'session.timing') self.sessionStdoutLog = self.logdir+'session.stdout' self.sessionHistoryLog = self.logdir+'session.history' self.programPath, self.programName=os.path.split(program) self.dar = self.programPath+'/dar2' infoOut( 'dar executable is '+self.dar) (out, stat) = self.s.run(self.dar+' -v ') # display dar version if stat != "0": message="""dar executable failed Please check DAR availability and configuration""" self.error(message) infoOut("current dar version is: "+out[len(out)-1][:-1]) self.cvsbase = ':pserver:[email protected]:/cvs_server/repositories/' infoOut('cvs server basename is ' + self.cvsbase ) # Initialize required attributes: self.numExe = 0 self.executables = [] self.applList = [] # allow zero applications self.geomFilesList = [] self.geomPathList = [] self.darHooks = {} self.DARRemoveTools = [] self.DARIgnoreFile = None # Log current time self.timingLog('RefDBDAR configured')
def saveAndQuit(self, darFile = 'useDefault'): """ Creates the final DARball and returns. This is called when the user is satisfied with the result, or when the criteria are satisfied (if in non-interactive mode). If successful, it will return the name of the final darball that was created, otherwise it will return 'None'. """ if darFile == 'useDefault': darFile = self.blddir + "/" +\ self.darMeta.getVersionTag() +\ "." + self.currentRequest.getArchitecture() +\ self.darballSuffix else: darFile = self.blddir + "/" +\ self.currentRequest.getDarName() +\ self.darballSuffix tarCmd = string.join ( [ 'tar -cz -f', darFile, '-C', os.path.join(self.blddir, self.dar.getTopInstDir()), getDarDirName(), '-C', self.blddir, self.darMeta.getVersionTag() ]) infoOut("Creating DAR distribution in " + self.blddir + "using tar command:") infoOut(tarCmd) (status, out) = commands.getstatusoutput(tarCmd) if status == 0: # successfull infoOut( "Created " + darFile) self.timingLog('Created darball: ' + darFile) return darFile else: infoOut( "Creation of dar file failed!\n " + out) return None
def getReleaseTop(projectName, releaseName): """Returns the location of the release installation""" # First try scramv1 list as more selective: infoOut("looking for " + projectName + " " + releaseName + " release top") cmd = 'scramv1 list -c ' + projectName infoOut("Trying " + cmd + " ... ") for line in popen2.popen2(cmd)[0].readlines(): proj, rel, loc = string.split(line) if (proj, rel) == (projectName, releaseName): return loc # Now lookup all project's releases using scram: cmd = 'scram listcompact ' + projectName infoOut("Trying " + cmd + " ... ") for line in popen2.popen2(cmd)[0].readlines(): proj, rel, loc = string.split(line) if (proj, rel) == (projectName, releaseName): return loc # If we still did not find a proper release, it must be user's error. sys.exit("SCRAM2DAR.getReleaseTop: could not find " + str(releaseName) + \ " release. \n Hints: check your request and scram architecture.")
def parseRequest(self,file): # Disable try: if 0 == 0 : #try: f = open(file,'r') input = {} for line in f.readlines(): process(line, input) # Store input values from the request in attributes # (mapping to the RefDB used variable names): self.project = input['Application'] self.release = input['Version' ] # Handle multiple geometry files: if input.has_key("NumberOfGeometryFiles"): self.geomFileNum = int(input['NumberOfGeometryFiles']) for i in range(self.geomFileNum): self.geomFilesList.append(Geometry(input,str(i))) # DO check right away during parsing infoOut("Validating file "+self.geomFilesList[i].filePath +" ...") if not (self.geomFilesList[i]).valid(): self.error("Invalid geometry file: "+self.geomFilesList[i].filePath) # Add paths to multiple Geometry versions. if input.has_key("NumberOfGeometryXMLversions"): self.geompath_num=int(input['NumberOfGeometryXMLversions']) for i in range(self.geompath_num): self.geompath_list.append(GeometryPath(input,str(i))) # Get darfile related info: self.darball=DARball(input) # an object # List of Executables to be searched in runtime PATH: if input.has_key("ExecutablesList"): self.executables=string.split(input['ExecutablesList'],";") # List of applications to be built from sources: if input.has_key("NumberOfExecutables"): self.numExe = int(input['NumberOfExecutables']) for i in range(self.numExe): if input.has_key('Application_'+str(i)): self.applList.append(Application(input,str(i)))# each an object if not self.numExe: raise "zero_num_exe" # Define a few DAR hooks that will extend scram runtime environment # by setting special DAR variables for name in input.keys(): # DAR_PRE_ hook: if name[0:8] == "DAR_PRE_": self.darHooks[name]=input[name] # DAR_POST_ hook: if name[0:9] == "DAR_POST_": self.darHooks[name]=input[name] # DAR_ROOTRC hook: if name=="DAR_ROOTRC": self.darHooks[name]=input[name] # DAR_IGNORE_FILE - a separate hook that helps to pass the file with the # runtime environment variables to be ignored in dar: if name == "DAR_ignore_file": self.DAR_ignore_file = input[name] # get list of tools to be removed from the project # configuration before packaging: if name == "DAR_remove_tools": self.DAR_remove_tools = string.split(input[name],";") # For all SCRAM managed projects use DAR2: self.packager = DAR2Packager(self) self.timingLog('input file processed')
def __init__(self, darInput, pltf=None, cnf=None): """ __init__ Initializes DAR Manager apects common to both create and install mode. """ Lister.__init__(self) # Set defaults: self.darballSuffix = '_dar.tar.gz' # and initialize variables: self.baseBomDict = {} # need for incremental darball self.darpool = None # self.dar = None # need for creating darball self.config = cnf infoOut('Validating DAR configuration ...') if getVerbose(): print "CONFIGURATION:", self.config # Check that dar shared pool is set up, exists and has right permissions: # DAR shared pool is to lookup for a base dar distributions and/or # installations . if self.config.has_key("dar_shared_pool"): self.sharedPool = self.config["dar_shared_pool"] else: sys.exit("ERROR:dar shared pool is not defined in the configuration") if self.config.has_key("dar_dist_pool"): self.distPool = self.config["dar_dist_pool"] else: sys.exit("ERROR: dar dist pool is not defined in the configuration") if self.config.has_key("dar_inst_pool"): self.instPool = self.config["dar_inst_pool"] else: sys.exit("ERROR: dar inst pool is not defined in the configuration") # Check that dar tmpdir is set, exists, and has right permissions: if self.config.has_key("dar_tmp_dir"): self.tmpdir = self.config["dar_tmp_dir"] else: sys.exit("ERROR: dar_tmp_dir is not defined in the configuration") if notWritable(self.tmpdir): sys.exit(notWritable(self.tmpdir)) # Each time when dar is called, it creates a new build directory # in tmpdir pool, using a unique datestamp: self.blddir = self.tmpdir+'/'+str(time.time()) # Start logger and pre-set log files: self.logdir = self.blddir + '/logfiles' # one dir for all logfiles self.timingLog = Logger(self.logdir+'/session.timing') self.sessionStdoutLog = self.logdir+'/session.stdout' self.sessionHistoryLog = self.logdir+'/session.history' # Create necessary directories and files: os.makedirs(self.blddir) os.makedirs(self.logdir) for logfile in (self.sessionStdoutLog, self.sessionHistoryLog): infoOut('Creating ' + logfile + ' file') open(logfile, 'w').close() # Get platform info: self.platform = pltf self.timingLog('Completed configuration and setup ') # Create a request self.currentRequest = Request(darInput, self.timingLog) self.timingLog('Created request object') # Initialize dar metadata self.darMeta = Metadata() self.darMeta.setDarVersion(getDarVersion()) # Get release/version metadata from request and # put them into Metadata container: self.darMeta.setDarInput(darInput) self.darMeta.setBaseReleaseName(self.currentRequest.getBaseReleaseName()) self.darMeta.setProjectName(self.currentRequest.getProjectName()) self.darMeta.setVersionTag(self.currentRequest.getVersionTag()) # Architecture self.darMeta.setArchitecture(self.currentRequest.getArchitecture())
def installApplication(self, installDir, testmode): """ Manager.installApplication Installs the application by performing the following steps: - checks to see if the installation directory is writable - loads metadata - checks if enough disk space is available - checks to see if this is an incremental installation - looks for the base disribution for this release - checks for previous installation of this package and checks the md5sum <not implemented> - unpacks the installation package - publishes package metadata - sets up the environment scripts and runs setup scripts - creates links and checks installation size """ if notWritable(installDir): sys.exit(notWritable(installDir)) # Absolutize path if needed: if not os.path.isabs(installDir): installDir = os.path.abspath(installDir) # Extract metadata from distribution: metadata = loadMetadata(getMetaDataFile(), archive = self.currentRequest.getDarFile()) infoOut('Loaded DAR metadata from '+self.currentRequest.getDarFile()) # If in test mode, print out users info and exit: if testmode: print metadata.userInfo() return # Check that there is enough space in the installation directory: available = spaceLeft(installDir) installSize = float(metadata.getInstallationSize()) if available < installSize: sys.exit('Not enough space on the disk!\n Installation size: ' \ + str(installSize) + ' KB\n Available: ' + str(available) + ' KB') self.timingLog('Checked space left in '+installDir) ########################################## # Handling incremental DARballs: ########################################## # Check if darball metadata contain a reference to a base dar: baseDar = metadata.getBaseDar() if baseDar: infoOut("This is incremental distribution based on "+baseDar) # This is an incremental darball, so # we need matching base installation. baseInstallation = self.currentRequest.baseDar if not baseInstallation: usageError ('Please specify a base .') # Lookup for base installation in the installation pool: # baseInstallation = self.findBaseInstallation(self.distPool, # baseDar) # if not baseInstallation: # sys.exit( 'Could not find base installation for ' + # baseDar + ' # in '+self.sharedPool) infoOut("(todo)Verifying base installation "+baseInstallation) ########################################## # General actions for all DARballs: ########################################## # Check if the installation already exists: releaseInstTop = os.path.join(installDir, metadata.getVersionTag(), metadata.getArchitecture()) if os.path.exists( releaseInstTop): # TODO: validate the installation using md5sum and # tell user the results sys.exit("ERROR: You already have installation here: \n " \ +releaseInstTop+"\nExiting ....\n") # Unpack darball infoOut('Unpacking '+self.currentRequest.getDarFile()+' .... ') unpackCommand = 'tar -xz -C ' + \ installDir + ' -f ' + \ self.currentRequest.getDarFile() (status, out) = commands.getstatusoutput(unpackCommand) # Check that in unpacked into toInstallDir as expected: if status: # failed if out: # save command output in the logfile: unpackLogfile = os.path.join(self.logdir, 'dar_unpack.log') tarlog = open(unpackLogfile, 'w') tarlog.write('Output from unpacking command:\n' + \ unpackCommand + '\n' + out ) tarlog.close() sys.exit ('Unpacking failed with exit status ' + status + \ '\nOutput can be found in \n' + unpackLogfile ) elif not os.path.isdir(releaseInstTop): sys.exit ('Can not find '+releaseInstTop) # Link to a base installation for incremental darballs if baseDar: infoOut("Create a link to base installation:\n ln -s " +baseInstallation+'/shared '+releaseInstTop+'/base') os.symlink(baseInstallation+'/shared', releaseInstTop+'/base') # Set up environment scripts: infoOut("Setting up the installation") templateStub = os.path.join(releaseInstTop, getSetupScriptBasename()) newSetupScriptStub = os.path.join(releaseInstTop, 'envSetup') helpText = self.updateSetupScripts(\ templateStub, \ releaseInstTop, \ newSetupScriptStub ) # For compatibility with the old Production tools: oldSetupScriptStub = os.path.join(releaseInstTop, metadata.getVersionTag() + '_env') self.updateSetupScripts(\ templateStub, \ releaseInstTop, \ oldSetupScriptStub ) # Move script templates to the DAR admin directory. #infoOut('Removing setup scripts templates ...') cmd = 'mv ' + templateStub + '.*sh' + ' ' \ + installDir + '/' + getDarDirName() (status, out)=commands.getstatusoutput(cmd) if status != 0: # did not succeed DARInternalError("In installApplication: " + "doing command" + cmd + "\ncommand output :\n" + out) #infoOut("(todo) Do md5sum check of BOM in resulting installation") #infoOut("(todo) If successful, " + # "register installation in publishing service ") self.publishMetadata(installDir + '/' + getDarDirName()) # Publish installation metadata: self.publishMetadata(installDir + '/' + getDarDirName()) #Print out runtime environment setup help (and exit): infoOut(helpText) infoOut("Installation completed.")
def addVariable(self, key, value): """ Request.addVariable: Add an environment variable to the list """ infoLine = "Add variable "+key+" = "+value+"\ntype=" # Variable types identified by name : if key == 'LD_LIBRARY_PATH': infoOut(infoLine+'LibPath') self.rteDict[key] = LibPath(key, value) return if key == 'DAR_runtime_PATH': infoOut(infoLine+'BinPath') self.rteDict[key] = BinPath(key, value) return if key == 'PATH': infoOut(infoLine+'BinPath') self.rteDict[key] = BinPath(key, value) return if key == 'SCRAMRT_PATH': infoOut(infoLine+'BinPath') self.rteDict[key] = BinPath(key, value) return # Variable types identified by value: if os.path.isfile(value): infoOut(infoLine+'PathToFile') self.rteDict[key] = PathToFile(key, value) return if os.path.isdir(value): infoOut(infoLine+'PathToDir') self.rteDict[key] = PathToDir(key, value) return if re.search(':', value): # probably a list of paths, but could also be a url, etc # just for test now treat all like directory paths. infoOut(infoLine+'PathToDir') self.rteDict[key] = PathToDir(key, value) return # The rest is considered a simple value type: infoOut(infoLine+'SimpleValue') self.rteDict[key] = SimpleValue(key, value)
def executeUse(self, cmd): """ Request.executeUse Executes actions based on a user's input 'use' keyword Commands are given as: use <cmd> <argument> <input> where cmd can be: scram - get our DAR input from an existing SCRAM file - if 'rte' is the argument given, we gather the runtime environment from a given relative or absolute path - if 'arch' is given as an argument, we use the scram-defined bintype dar - set some dar information - if 'base' is the input argument, we check to see if the input is an existing darball - if 'file' is the argument given, then we use the given file to get our input - if 'tagname' is given, we set the version tag - if 'projectname' is given, we set the project name """ if cmd.args[0] == "scram": if cmd.args[1] == "rte": # Get release location, information, etc if os.path.isabs(cmd.args[2]): # One can specify top release directory with # absolute path. # Examples: # use scram rte /afs/cern.ch/cms/Releases/ORCA/ORCA_8_4_0 # use scram rte /home/user/work/ORCA_8_4_0 location = cmd.args[2] else: #Or it can be release name preceeded by a project name: # use scram rte ORCA ORCA_8_4_0 #In this case dar gets release top from the scram #database: location = getReleaseTop(cmd.args[2], cmd.args[3]) if not os.path.isdir(location): cmd.help() message = "could not find release, check command syntax!" raise InputError(cmd.line, message) # Choose scram executable name according to scram version used # in the project. majorVers = getScramVersion(location)[:-1] if majorVers[:2] == "V1": self.scram = SCRAM("scramv1") elif majorVers[:2] == "V0": self.scram = SCRAM("scram") else: sys.exit("""ERROR! do not know how to define scram executable for scram version """ + majorVers) # Set attributes and additional dar input that scram can get # from this location: relMeta = getReleaseMetadata (location) self.setProjectName (getProjectName (relMeta)) self.setVersionTag (getVersionTag (relMeta)) self.setBaseReleaseName (getBaseReleaseName(relMeta)) extraDarInput = self.scram.generateDarInput(location) # Process scram generated rte same way as variables # from darInput: for line in extraDarInput: # process environment variables: result = avariable.match(line) if result: vName, vValue = result.groups() # discard surrounding whitespaces: vName = string.strip(vName) vValue = string.strip(vValue) # Creates a variable object and puts into self.rteDict: self.addVariable(vName, vValue) continue # process commands ( variables ignored by # default in SCRAM2DAR): extraCmd = DARcommand(line) self.cmdList.append(extraCmd) if extraCmd.name == "ignore": self.executeIgnore(extraCmd) continue # Check if first command argument is existing variable name. # Commands associated with environment variable # will use corresponding variable method: if self.rteDict.has_key(extraCmd.args[0]): var = self.rteDict[extraCmd.args[0]] self.executeVarCommand(extraCmd, var) continue self.setArchitecture(self.scram.platf) # use scram-defined bintype if cmd.args[1] == "arch": # TO DO: use extra argument to set non-default architecture, # e.g. "use scram arch <some arch>". print "WARNING: Not implemented command: use scram arch" elif cmd.args[0] == "dar": if cmd.args[1] == "base": # Command syntax : use dar base <base dar ball> self.baseDar = cmd.args[2] # Do quick checks just to make sure baseDar could be a dar; # return immediately if base distribution is not found: # It can be dar installation directory or a dar # distribution file: if os.path.isdir(self.baseDar): infoOut('Use Base DAR installation from a directory:'+\ self.baseDar) elif os.path.isfile(self.baseDar): infoOut('Use Base DAR distribution from file:'+\ self.baseDar) else: # Lookup for base darball in the pool infoOut('Base DAR is not a file or a directory: '+\ self.baseDar) infoOut('Will be looked up later in the dar shared pool ') if cmd.args[1] == 'file': # Command syntax : use dar file <darball path (or LFN?)> self.setDarFile(cmd.args[2]) if cmd.args[1] == 'tagname': # Command syntax : use dar tagname <version tag> # TODO: add a check that tag contains allowed characters only self.setVersionTag (cmd.args[2]) elif cmd.args[1] == 'projectname': # Command syntax : use dar projectname <project tag> # TODO: add a check that project name contains # allowed characters only self.setProjectName (cmd.args[2]) if cmd.args[1] == "arch": # use extra argument to set non-default architecture, # e.g. "use scram arch <some arch>". infoOut('set Architecture to ' + cmd.args[2]) self.setArchitecture(cmd.args[2]) else: warning('unrecognized command syntax: ', cmd.line)
def __init__(self, darInput, logger): Lister.__init__(self) # base constructor # Store RTE variables in a dictionary: # no two variables can have same name. self.logger = logger self.rteDict = {} self.cmdList = [] self._ReleaseMetadata = {} self._BaseReleaseName = None self._ProjectName = 'undefined' self._VersionTag = 'undefined' self._DarFile = '' # By default we don't know anything about scram: self.scram = None self.baseDar = None self._Arch = 'arch' ################################## # Fill up rteDict and cmdList ################################## infoOut('Reading lines from dar input:') for line in darInput: # Discard comments and surrounding whitespaces: line = re.sub('\t*#.*', '', line) line = string.strip(line) #################### # Skip blank lines #################### if blankLine.match(line): continue ########################################## # Process variable settings ############################################ result = avariable.match(line) if result: vName, vValue = result.groups() # discard surrounding whitespaces: vName = string.strip(vName) vValue = string.strip(vValue) # Creates a variable object and puts into self.rteDict: self.addVariable(vName, vValue) continue ################################## # Consider the rest as commands: ########################################## cmd = DARcommand(line) self.cmdList.append(cmd) # Execute commands: if cmd.name == "ignore": self.executeIgnore(cmd) continue if cmd.name == "use": self.executeUse(cmd) continue # next line # Check if first command argument is existing variable name. # Commands associated with environment variable # will use corresponding variable method: if self.rteDict.has_key(cmd.args[0]): var = self.rteDict[cmd.args[0]] self.executeVarCommand(cmd, var) continue # next line # Check if first command argument is "ALL", and if it is, # execute corresponding method for all variables. if cmd.args[0] == "ALL": self.executeALLVariables(cmd) ########################################### # Provide hooks to override PATH settings, in order to control # which executables are to be taken into the distribution. if self.scram: # i.e. we deal with a scram managed project # In scram V0 SCRAMRT_PATH contains additions to PATH, so we use it # to override PATH value: if self.scram.program == "scram": if self.rteDict.has_key('SCRAMRT_PATH'): tmp = self.rteDict['SCRAMRT_PATH'].getValue() self.rteDict['PATH'].setValue(tmp) del tmp # Same for LD_LIBRARY_PATH: if self.rteDict.has_key('SCRAMRT_LD_LIBRARY_PATH'): tmp = self.rteDict['SCRAMRT_LD_LIBRARY_PATH'].getValue() self.rteDict['LD_LIBRARY_PATH'].setValue(tmp) del tmp # In scram V1 SCRAMRT_PATH contains default user's PATH settings, # so we strip it from the PATH value: if self.scram.program == "scramv1": tmp = self.rteDict['PATH'].getValue() self.rteDict['PATH'].setValue(tmp.replace(':$SCRAMRT_PATH','')) del tmp # Similar for LD_LIBRARY_PATH: tmp = self.rteDict['LD_LIBRARY_PATH'].getValue() tmp = tmp.replace(':$SCRAMRT_LD_LIBRARY_PATH','') self.rteDict['LD_LIBRARY_PATH'].setValue(tmp) del tmp # DAR_runtime_PATH overrides both PATH and SCRAMRT_PATH # (independent of scram version): if self.rteDict.has_key('DAR_runtime_PATH'): tmp = self.rteDict['DAR_runtime_PATH'].getValue() self.rteDict['PATH'].setValue(tmp) del tmp