def setReplicaFlag(self, arg): """ Setter """ if arg.lower() in ('yes', 'no', 'all'): self.options['ReplicaFlag'] = arg else: gLogger.error('Unknown replica flag: %s' % arg) return DIRAC.exit(1) return DIRAC.S_OK()
def setVisibility(self, arg): """ Setter """ if arg.lower() in ('yes', 'no', 'all'): self.options['Visibility'] = arg else: gLogger.error('Unknown visibility flag: %s' % arg) return DIRAC.exit(1) return DIRAC.S_OK()
def setProxyLifeTime(self, arg): try: fields = [f.strip() for f in arg.split(":")] self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60 except ValueError: print "Can't parse %s time! Is it a HH:MM?" % arg return DIRAC.S_ERROR("Can't parse time argument") return DIRAC.S_OK()
def listtable(self, entries): for i in xrange(1, entries + 1): retDict = self._getFields('MyDB_testTable', [], inFields=['Status'], inValues=[i]) if not retDict['OK']: return retDict return DIRAC.S_OK(i)
def setServer(optionValue): global configurationServer configurationServer = optionValue DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers', configurationServer) DIRAC.gConfig.setOptionValue(cfgInstallPath('ConfigurationServer'), configurationServer) return DIRAC.S_OK()
def setSoftwarePackage(self, softwarePackage=None): """ Define the software package or try to get it from the job.info file This method can be used when executing outside a Workflow """ if softwarePackage: self.softwarePackage = softwarePackage return DIRAC.S_OK() try: infoFile = open('job.info') for line in infoFile.readlines(): if line.find('SoftwarePackages') > -1: self.softwarePackage = line.split('=')[-1].strip() break return DIRAC.S_OK() except Exception: return DIRAC.S_ERROR('Can not set softwarePackage')
def _checkemptydir(self, path): """ check that the directory is not empty """ if len(glob.glob(path)) == 0: error = 'Empty directory: %s' % path return DIRAC.S_ERROR(error) else: return DIRAC.S_OK()
def setProxyLifeTime(self, arg): try: fields = [f.strip() for f in arg.split(":")] self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60 except BaseException: gLogger.notice("Can't parse %s time! Is it a HH:MM?" % arg) return DIRAC.S_ERROR("Can't parse time argument") return DIRAC.S_OK()
def setSiteName(optionValue): global siteName siteName = optionValue DIRAC.gLogger.debug('/LocalSite/Site =', siteName) Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName) DIRAC.__siteName = False DIRAC.gConfig.setOptionValue(cfgInstallPath('SiteName'), siteName) return DIRAC.S_OK()
def clean_output_file(output_files): """ Delete Local Files """ DIRAC.gLogger.info('Deleting Local Files') for afile in output_files: DIRAC.gLogger.warn('Remove local File %s' % afile) os.remove(afile) return DIRAC.S_OK()
def installDIRACScripts(self, package_dir): """ copy prod3 DIRAC scripts in the current directory """ cmd = 'cp ' + os.path.join(package_dir, 'dirac_*') + ' .' if not os.system(cmd): return DIRAC.S_OK() else: return DIRAC.S_ERROR('Failed to install DIRAC scripts')
def setServer(optionValue): global configurationServer configurationServer = optionValue Script.localCfg.addDefaultEntry('/DIRAC/Configuration/Servers', configurationServer) DIRAC.gConfig.setOptionValue(cfgInstallPath('ConfigurationServer'), configurationServer) return DIRAC.S_OK()
def putAndRegisterPROD3(args): """ simple wrapper to put and register all PROD3 files Keyword arguments: args -- a list of arguments in order [] """ metadata = args[0] metadatafield = args[1] filemetadata = args[2] inputpath = args[3] basepath = args[4] start_run_nb = args[5] catalogs = ['DIRACFileCatalog'] # # Create MD structure prod3dm = Prod3DataManager(catalogs) res = prod3dm.createMDStructure(metadata, metadatafield, basepath, 'tel_sim') if res['OK']: path = res['Value'] else: return res # # Upload data files datadir = os.path.join(inputpath, 'Data/*') res = prod3dm._checkemptydir(datadir) if not res['OK']: return res # ## Add start_run_number to run_number and update filemetadata fmd = json.loads(filemetadata) run_number = int(fmd['runNumber']) + int(start_run_nb) fmd['runNumber'] = '%08d' % run_number fmdjson = json.dumps(fmd) # ## Get the run directory # runpath = prod3dm._getRunPath( fmdjson ) runpath = prod3dm._getRunPath(fmd) for localfile in glob.glob(datadir): filename = os.path.basename(localfile) lfn = os.path.join(path, 'Data', runpath, filename) res = prod3dm.putAndRegister(lfn, localfile, fmdjson) if not res['OK']: return res # ## Upload log files tarname = filename.split('___cta-prod3')[0] + '.tar.gz' res = prod3dm.createTarLogFiles(inputpath, tarname) if not res['OK']: return DIRAC.S_ERROR('prod3dm.createTarLogFiles failed') lfn = os.path.join(path, 'Log', runpath, tarname) # res = prod3dm.putAndRegister( lfn, tarname, filemetadata ) res = prod3dm.putAndRegister(lfn, tarname, fmdjson) if not res['OK']: return res return DIRAC.S_OK()
def checkRecord(self): result = self.getValue("ExecTime") if not result["OK"]: return result execTime = result["Value"] if execTime > 33350400: # 1 year return DIRAC.S_ERROR( "OOps. More than 1 year of cpu time smells fishy!") return DIRAC.S_OK()
def dumpSetupScriptPath(self, package_dir, textfilename='setup_script_path.txt'): """ dump the path to setupPackage.sh in a one line ascii file to be read and source by the following script """ script_path = os.path.join(package_dir, 'setupPackage.sh') open(textfilename, 'w').writelines(script_path + '\n') return DIRAC.S_OK()
def createCorsikaFileSystAndMD(): # before creating indexes, it would be fine to know what are those that already exist in the DB # Creating INDEXES in DFC DB corsikaDirMDFields = {} corsikaDirMDFields['prodName'] = 'VARCHAR(128)' corsikaDirMDFields['thetaP'] = 'float' corsikaDirMDFields['phiP'] = 'float' corsikaDirMDFields['altitude'] = 'float' corsikaDirMDFields['particle'] = 'VARCHAR(128)' corsikaDirMDFields['energyInfo'] = 'VARCHAR(128)' corsikaDirMDFields['viewCone'] = 'float' corsikaDirMDFields['corsikaProdVersion'] = 'VARCHAR(128)' corsikaDirMDFields['nbShowers'] = 'int' corsikaDirMDFields['outputType'] = 'VARCHAR(128)' corsikaDirMDFields['runNumSeries'] = 'int' createIndexes(corsikaDirMDFields) # Adding Directory level metadata Values to DFC corsikaDirMD = {} corsikaDirMD['thetaP'] = thetaP corsikaDirMD['phiP'] = phiP corsikaDirMD['altitude'] = obslev corsikaDirMD['energyInfo'] = energyInfo corsikaDirMD['corsikaProdVersion'] = corsikaProdVersion res = createDirAndInsertMD(corsikaDirPath, corsikaDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR('Problem creating Corsika Directory MD ') corsikaParticleDirMD = {} corsikaParticleDirMD['particle'] = particle corsikaParticleDirMD['viewCone'] = viewCone res = createDirAndInsertMD(corsikaParticleDirPath, corsikaParticleDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR('Problem creating Corsika Particle Directory MD ') corsikaParticleDataDirPath = os.path.join(corsikaParticleDirPath, 'Data') corsikaParticleDataDirMD = {} corsikaParticleDataDirMD['outputType'] = 'corsikaData' res = createDirAndInsertMD(corsikaParticleDataDirPath, corsikaParticleDataDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR( 'Problem creating Corsika Particle Data Directory MD ') corsikaParticleLogDirPath = os.path.join(corsikaParticleDirPath, 'Log') corsikaParticleLogDirMD = {} corsikaParticleLogDirMD['outputType'] = 'corsikaLog' res = createDirAndInsertMD(corsikaParticleLogDirPath, corsikaParticleLogDirMD) if res != DIRAC.S_OK: return DIRAC.S_ERROR( 'Problem creating Corsika Particle Log Directory MD ') return DIRAC.S_OK('Corsika Directory MD successfully created')
def cleanLocalFiles ( self, datadir, pattern = '*' ): """ remove files matching pattern in datadir """ for localfile in glob.glob( os.path.join( datadir, pattern ) ): DIRAC.gLogger.notice( 'Removing local file: ', localfile ) os.remove( localfile ) return DIRAC.S_OK()
def checkSoftwarePackage(package, area): """ Check if the requested version of package is installed under the given area """ fileName = _getEnvFileName(package, area) softwareDir = _getSoftwareDir(package, area) try: gLogger.notice('Looking for Environment file', fileName) if os.path.exists(fileName): return DIRAC.S_OK() else: gLogger.notice('Looking for Software dir', softwareDir) if os.path.isdir(softwareDir): return DIRAC.S_OK() except Exception: pass return DIRAC.S_ERROR()
def removeSoftwarePackage(self, packagedir): """ remove Software Package """ cmd = 'rm -Rf ' + packagedir if not os.system(cmd): return DIRAC.S_OK() else: error = 'Failed to remove %s' % packagedir return DIRAC.S_ERROR(error)
def checktable(self): retDict = self._update('DROP TABLE IF EXISTS `MyDB_testTable`') if not retDict['OK']: return retDict retDict = self._update('CREATE TABLE `MyDB_testTable` ( ' '`ID` INTEGER NOT NULL AUTO_INCREMENT, ' '`LastUpdate` TIMESTAMP, ' '`Status` varchar(128), ' 'PRIMARY KEY (`ID`) )') if not retDict['OK']: return retDict return DIRAC.S_OK()
def install_dirac_scripts(self, package_dir): """ copy DIRAC scripts in the current directory """ dirac_scripts = glob.glob(os.path.join(package_dir, 'dirac_*')) try: for one_file in dirac_scripts: shutil.copy2(one_file, '.') return DIRAC.S_OK() except shutil.Error as error: return DIRAC.S_ERROR('Failed to install DIRAC scripts:\n%s'%error)
def _getSharedArea(self): """ get Shared Area """ if os.environ.has_key(self.SW_SHARED_DIR): shared_area = os.path.join(os.environ[self.SW_SHARED_DIR], 'software') else: return DIRAC.S_ERROR('Shared area not found') return DIRAC.S_OK(shared_area)
def setupSoftware(args): """ setup a given software package to be used in main Keyword arguments: args -- a list of arguments in order [package, version, arch] """ # check number of arguments if len(args) < 2: Script.showHelp() return DIRAC.S_ERROR('Wrong number of arguments') # get arguments package = args[0] version = args[1] program_category = 'simulations' if len(args) >= 3: program_category = args[2] arch = "sl6-gcc44" if len(args) == 4: arch = args[3] soft_category = {package: program_category} prod3swm = Prod3SoftwareManager(soft_category) # check if and where Package is installed res = prod3swm.checkSoftwarePackage(package, version, arch) if not res['OK']: res = prod3swm.installSoftwarePackage(package, version, arch) if not res['OK']: return res else: package_dir = res['Value'] prod3swm.dumpSetupScriptPath(package_dir) return DIRAC.S_OK() # # dump the SetupScriptPath to be sourced by DIRAC scripts # ## copy DIRAC scripts in the current directory package_dir = res['Value'] prod3swm.dumpSetupScriptPath(package_dir) res = prod3swm.installDIRACScripts(package_dir) if not res['OK']: return res return DIRAC.S_OK()
def __setLastLoadAndUptime(self, instanceID, load, uptime): if not uptime: result = self._query( "SELECT MAX( UNIX_TIMESTAMP( `Update` ) ) - MIN( UNIX_TIMESTAMP( `Update` ) ) FROM `vm_History` WHERE VMInstanceID = %d GROUP BY VMInstanceID" % instanceID) if result['OK'] and len(result['Value']) > 0: uptime = int(result['Value'][0][0]) sqlUpdate = "UPDATE `vm_Instances` SET `Uptime` = %d, `Load` = %f WHERE `VMInstanceID` = %d" % ( uptime, load, instanceID) self._update(sqlUpdate) return DIRAC.S_OK()
def execute(self): """ The method called by the Workflow framework """ from DIRAC.Core.Utilities.Subprocess import systemCall from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea ret = self.__checkInputs() if not ret['OK']: return ret ret = getSoftwareEnviron(self.softwarePackage) if not ret['OK']: error = ret['Message'] self.log.error(error, self.softwarePackage) return DIRAC.S_ERROR(' '.join([error, str(self.softwarePackage)])) hapEnviron = ret['Value'] hessroot = hapEnviron['HESSROOT'] rootlogon_file = hessroot + '/rootlogon.C' cp_cmd = 'cp ' + rootlogon_file + ' .' os.system(cp_cmd) fileName = hessroot + self.rootMacro if fileName[-1] == '+': # If the macro has to be compiled there is an extra "+" at the end of its name fileName = fileName[:-1] if not os.path.isfile(fileName): error = 'Hap Root macro file does not exist:' self.log.error(error, fileName) return DIRAC.S_ERROR(' '.join([error, fileName])) fileName = hessroot + self.rootMacro cmdTuple = ['root', '-b', '-q'] cmdTuple += [ '%s( %s )' % (fileName, ', '.join(self.rootArguments).replace("'", '"')) ] self.log.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, self.sendOutput, env=hapEnviron) if not ret['OK']: self.log.error('Failed to execute Root:', ret['Message']) return DIRAC.S_ERROR('Can not execute Hap root macro') return DIRAC.S_OK()
def __checkInputs(self): """ Check the data members for the expected Input Parameters """ for inputPar in INPUTPARAMETERS: if not inputPar in dir(self): error = 'Input Parameter not defined:' self.log.error(error, inputPar) return DIRAC.S_ERROR(' '.join([error, inputPar])) return DIRAC.S_OK()
def setDirectory(self, arg): """ Setter """ if os.path.exists(arg) and not os.path.isdir(arg): with open(arg, 'r') as inFile: directories = [ line.split()[0] for line in inFile.read().splitlines() if line.strip() ] else: directories = arg.split(',') self.options.setdefault('Directory', set()).update(directories) return DIRAC.S_OK()
def setIssuer(self, optionValue): # If the user selects the issuer, it means that there will be authorization through tokens os.environ["DIRAC_USE_ACCESS_TOKEN"] = "True" # Allow the user to enter only the server domain if not optionValue.startswith("http://") or not optionValue.startswith( "https://"): optionValue = f"https://{optionValue.lstrip('/')}" if len(optionValue.strip("/").split("/")) == 3: optionValue = f"{optionValue.rstrip('/')}/auth" self.issuer = optionValue DIRAC.gConfig.setOptionValue("/DIRAC/Security/Authorization/issuer", self.issuer) return DIRAC.S_OK()
def downloadFile(lfn): """ Download a file using DMS Keyword arguments: lfn -- a logical file name """ DIRAC.gLogger.info('Downloading ',lfn) dm = DataManager() res=dm.getFile(lfn) if not res['OK']: DIRAC.gLogger.error ( res['Message'] ) DIRAC.gLogger.error ( 'Could not download %s'%lfn ) DIRAC.exit( -1 ) return DIRAC.S_OK()
def setProductions(self, arg): """ Parse production numbers """ prods = [] if arg.upper() == "ALL": self.options['Productions'] = arg return DIRAC.S_OK() try: for prod in arg.split(','): if prod.find(":") > 0: pr = prod.split(":") for i in xrange(int(pr[0]), int(pr[1]) + 1): prods.append(i) else: prods.append(prod) self.options['Productions'] = [int(prod) for prod in prods] except ValueError as e: gLogger.error("Invalid production switch value: %s" % arg, repr(e)) self.options['Productions'] = ['Invalid'] return DIRAC.S_ERROR() return DIRAC.S_OK()