def findGenericPilotCredentials( vo = False, group = False ): if not group and not vo: return S_ERROR( "Need a group or a VO to determine the Generic pilot credentials" ) if not vo: vo = Registry.getVOForGroup( group ) if not vo: return S_ERROR( "Group %s does not have a VO associated" % group ) opsHelper = Operations.Operations( vo = vo ) pilotGroup = opsHelper.getValue( "Pilot/GenericPilotGroup", "" ) pilotDN = opsHelper.getValue( "Pilot/GenericPilotDN", "" ) if pilotDN and pilotGroup: gLogger.verbose( "Pilot credentials have been defined in the CS. Using %s@%s" % ( pilotDN, pilotGroup ) ) result = gProxyManager.userHasProxy( pilotDN, pilotGroup, 86400 ) if not result[ 'OK' ]: return S_ERROR( "%s@%s has no proxy uploaded to the ProxyManager" ) return S_OK( ( pilotDN, pilotGroup ) ) #Auto discover gLogger.verbose( "Pilot credentials are not defined. Autodiscovering..." ) if pilotGroup: pilotGroups = [ pilotGroup ] else: result = Registry.getGroupsWithProperty( Properties.GENERIC_PILOT ) if not result[ 'OK' ]: return result pilotGroups = [] groups = result[ 'Value' ] if not groups: return S_ERROR( "No group with %s property defined" % Properties.GENERIC_PILOT ) result = Registry.getGroupsForVO( vo ) if not result[ 'OK' ]: return result for voGroup in result[ 'Value' ]: if voGroup in groups: pilotGroups.append( voGroup ) if not pilotGroups: return S_ERROR( "No group for VO %s is a generic pilot group" % vo ) for pilotGroup in pilotGroups: DNs = Registry.getDNsInGroup( pilotGroup ) if not DNs: continue if pilotDN: if pilotDN not in DNs: continue result = gProxyManager.userHasProxy( pilotDN, pilotGroup, 86400 ) if result[ 'OK' ] and result[ 'Value' ]: gLogger.verbose( "Discovered pilot credentials are %s@%s" % ( pilotDN, pilotGroup ) ) return S_OK( ( pilotDN, pilotGroup ) ) else: for DN in DNs: result = gProxyManager.userHasProxy( DN, pilotGroup, 86400 ) if result[ 'OK' ] and result[ 'Value' ]: gLogger.verbose( "Discovered pilot credentials are %s@%s" % ( DN, pilotGroup ) ) return S_OK( ( DN, pilotGroup ) ) if pilotDN: return S_ERROR( "DN %s does not have group %s" % ( pilotDN, pilotGroups ) ) return S_ERROR( "No generic proxy in the Proxy Manager with groups %s" % pilotGroups )
def findGenericPilotCredentials( vo = False, group = False ): if not group and not vo: return S_ERROR( "Need a group or a VO to determine the Generic pilot credentials" ) if not vo: vo = Registry.getVOForGroup( group ) if not vo: return S_ERROR( "Group %s does not have a VO associated" % group ) opsHelper = Operations.Operations( vo = vo ) pilotGroup = opsHelper.getValue( "Pilot/GenericPilotGroup", "" ) pilotDN = opsHelper.getValue( "Pilot/GenericPilotDN", "" ) if pilotDN and pilotGroup: gLogger.verbose( "Pilot credentials from CS: %s@%s" % ( pilotDN, pilotGroup ) ) result = gProxyManager.userHasProxy( pilotDN, pilotGroup, 86400 ) if not result[ 'OK' ]: return S_ERROR( "%s@%s has no proxy in ProxyManager" ) return S_OK( ( pilotDN, pilotGroup ) ) #Auto discover gLogger.verbose( "Pilot credentials are not defined. Autodiscovering..." ) if pilotGroup: pilotGroups = [ pilotGroup ] else: result = Registry.getGroupsWithProperty( Properties.GENERIC_PILOT ) if not result[ 'OK' ]: return result pilotGroups = [] groups = result[ 'Value' ] if not groups: return S_ERROR( "No group with %s property defined" % Properties.GENERIC_PILOT ) result = Registry.getGroupsForVO( vo ) if not result[ 'OK' ]: return result for voGroup in result[ 'Value' ]: if voGroup in groups: pilotGroups.append( voGroup ) if not pilotGroups: return S_ERROR( "No generic pilot group for VO %s" % vo ) for pilotGroup in pilotGroups: DNs = Registry.getDNsInGroup( pilotGroup ) if not DNs: continue if pilotDN: if pilotDN not in DNs: continue result = gProxyManager.userHasProxy( pilotDN, pilotGroup, 86400 ) if result[ 'OK' ] and result[ 'Value' ]: gLogger.verbose( "Discovered pilot credentials: %s@%s" % ( pilotDN, pilotGroup ) ) return S_OK( ( pilotDN, pilotGroup ) ) else: for DN in DNs: result = gProxyManager.userHasProxy( DN, pilotGroup, 86400 ) if result[ 'OK' ] and result[ 'Value' ]: gLogger.verbose( "Discovered pilot credentials: %s@%s" % ( DN, pilotGroup ) ) return S_OK( ( DN, pilotGroup ) ) if pilotDN: return S_ERROR( "DN %s does not have group %s" % ( pilotDN, pilotGroups ) ) return S_ERROR( "No generic proxy in the Proxy Manager with groups %s" % pilotGroups )
def __checkIfProxyUploadIsRequired( self ): result = gProxyManager.userHasProxy( self.ownerDN, self.ownerGroup, validSeconds = 18000 ) if not result[ 'OK' ]: gLogger.error( "Can't check if the user has proxy uploaded:", result[ 'Message' ] ) return True #Check if an upload is required return result[ 'Value' ] == False
def __checkIfProxyUploadIsRequired(self): result = gProxyManager.userHasProxy(self.ownerDN, self.ownerGroup, validSeconds=18000) if not result['OK']: gLogger.error("Can't check if the user has proxy uploaded:", result['Message']) return True # Check if an upload is required return not result['Value']
def findGenericPilotCredentials(vo=False, group=False, pilotDN='', pilotGroup=''): """ Looks into the Operations/<>/Pilot section of CS to find the pilot credentials. Then check if the user has a registered proxy in ProxyManager. if pilotDN or pilotGroup are specified, use them """ if not group and not vo: return S_ERROR("Need a group or a VO to determine the Generic pilot credentials") if not vo: vo = Registry.getVOForGroup(group) if not vo: return S_ERROR("Group %s does not have a VO associated" % group) opsHelper = Operations.Operations(vo=vo) if not pilotGroup: pilotGroup = opsHelper.getValue("Pilot/GenericPilotGroup", "") if not pilotDN: pilotDN = opsHelper.getValue("Pilot/GenericPilotDN", "") if not pilotDN: pilotUser = opsHelper.getValue("Pilot/GenericPilotUser", "") if pilotUser: result = Registry.getDNForUsername(pilotUser) if result['OK']: pilotDN = result['Value'][0] if pilotDN and pilotGroup: gLogger.verbose("Pilot credentials: %s@%s" % (pilotDN, pilotGroup)) result = gProxyManager.userHasProxy(pilotDN, pilotGroup, 86400) if not result['OK']: return S_ERROR("%s@%s has no proxy in ProxyManager") return S_OK((pilotDN, pilotGroup)) if pilotDN: return S_ERROR("DN %s does not have group %s" % (pilotDN, pilotGroup)) return S_ERROR("No generic proxy in the Proxy Manager with groups %s" % pilotGroup)
def findGenericCloudCredentials(vo=False, group=False): if not group and not vo: return S_ERROR( "Need a group or a VO to determine the Generic cloud credentials") if not vo: vo = Registry.getVOForGroup(group) if not vo: return S_ERROR("Group %s does not have a VO associated" % group) opsHelper = Operations.Operations(vo=vo) cloudGroup = opsHelper.getValue("Cloud/GenericCloudGroup", "") cloudDN = opsHelper.getValue("Cloud/GenericCloudDN", "") if not cloudDN: cloudUser = opsHelper.getValue("Cloud/GenericCloudUser", "") if cloudUser: result = Registry.getDNForUsername(cloudUser) if result['OK']: cloudDN = result['Value'][0] if cloudDN and cloudGroup: gLogger.verbose("Cloud credentials from CS: %s@%s" % (cloudDN, cloudGroup)) result = gProxyManager.userHasProxy(cloudDN, cloudGroup, 86400) if not result['OK']: return S_ERROR("%s@%s has no proxy in ProxyManager") return S_OK((cloudDN, cloudGroup)) return S_ERROR("Cloud credentials not found")
def findGenericCloudCredentials(vo=False, group=False): """Get the cloud credentials to use for a specific VO and/or group.""" if not group and not vo: return S_ERROR( "Need a group or a VO to determine the Generic cloud credentials") if not vo: vo = Registry.getVOForGroup(group) if not vo: return S_ERROR("Group %s does not have a VO associated" % group) opsHelper = Operations.Operations(vo=vo) cloudGroup = opsHelper.getValue("Cloud/GenericCloudGroup", "") cloudDN = opsHelper.getValue("Cloud/GenericCloudDN", "") if not cloudDN: cloudUser = opsHelper.getValue("Cloud/GenericCloudUser", "") if cloudUser: result = Registry.getDNForUsername(cloudUser) if result["OK"]: cloudDN = result["Value"][0] else: return S_ERROR("Failed to find suitable CloudDN") if cloudDN and cloudGroup: gLogger.verbose("Cloud credentials from CS: %s@%s" % (cloudDN, cloudGroup)) result = gProxyManager.userHasProxy(cloudDN, cloudGroup, 86400) if not result["OK"]: return result return S_OK((cloudDN, cloudGroup)) return S_ERROR("Cloud credentials not found")
def __checkIfProxyUploadIsRequired(self): """Check if an upload is required :return: bool """ result = gProxyManager.userHasProxy(self.ownerDN, self.ownerGroup, validSeconds=18000) if not result["OK"]: self.log.error("Can't check if the user has proxy uploaded", result["Message"]) return True # Check if an upload is required return not result["Value"]
def checkProxyUploaded(self, userDN, userGroup, requiredTime): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True, 'Value' : True/False } :param userDN: User DN :type userDN: string :param userGroup: DIRAC Group :type userGroup: string :param requiredTime: Required life time of the uploaded proxy :type requiredTime: boolean :return: S_OK,S_ERROR """ return gProxyManager.userHasProxy(userDN, userGroup, requiredTime)
def checkProxyUploaded( self, userDN, userGroup, requiredTime ): """Set the persistence of a proxy in the Proxy Manager Example usage: >>> print diracAdmin.setProxyPersistency( 'some DN', 'dirac group', True ) {'OK': True, 'Value' : True/False } @param userDN: User DN @type userDN: string @param userGroup: DIRAC Group @type userGroup: string @param requiredTime: Required life time of the uploaded proxy @type requiredTime: boolean @return: S_OK,S_ERROR """ return gProxyManager.userHasProxy( userDN, userGroup, requiredTime )
def findGenericPilotCredentials(vo=False, group=False, pilotDN="", pilotGroup=""): """Looks into the Operations/<>/Pilot section of CS to find the pilot credentials. Then check if the user has a registered proxy in ProxyManager. if pilotDN or pilotGroup are specified, use them :param str vo: VO name :param str group: group name :param str pilotDN: pilot DN :param str pilotGroup: pilot group :return: S_OK(tuple)/S_ERROR() """ if not group and not vo: return S_ERROR( "Need a group or a VO to determine the Generic pilot credentials") if not vo: vo = Registry.getVOForGroup(group) if not vo: return S_ERROR("Group %s does not have a VO associated" % group) opsHelper = Operations.Operations(vo=vo) if not pilotGroup: pilotGroup = opsHelper.getValue("Pilot/GenericPilotGroup", "") if not pilotDN: pilotDN = opsHelper.getValue("Pilot/GenericPilotDN", "") if not pilotDN: pilotUser = opsHelper.getValue("Pilot/GenericPilotUser", "") if pilotUser: result = Registry.getDNForUsername(pilotUser) if result["OK"]: pilotDN = result["Value"][0] if pilotDN and pilotGroup: gLogger.verbose("Pilot credentials: %s@%s" % (pilotDN, pilotGroup)) result = gProxyManager.userHasProxy(pilotDN, pilotGroup, 86400) if not result["OK"]: return S_ERROR("%s@%s has no proxy in ProxyManager") return S_OK((pilotDN, pilotGroup)) if pilotDN: return S_ERROR("DN %s does not have group %s" % (pilotDN, pilotGroup)) return S_ERROR("No generic proxy in the Proxy Manager with groups %s" % pilotGroup)
def submitNewBigJob( self ): result = jobDB.getJobJDL( str( self.__jobID ) , True ) classAdJob = ClassAd( result['Value'] ) executableFile = "" if classAdJob.lookupAttribute( 'Executable' ): executableFile = classAdJob.getAttributeString( 'Executable' ) tempPath = self.__tmpSandBoxDir dirac = Dirac() if not os.path.exists( tempPath ): os.makedirs( tempPath ) settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath ) self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir ) moveData = self.__tmpSandBoxDir + "/InputSandbox" + str( self.__jobID ) HiveV1Cli = HiveV1Client( self.__User , self.__publicIP ) returned = HiveV1Cli.dataCopy( moveData, self.__tmpSandBoxDir ) self.log.info( 'Copy the job contain to the Hadoop Master with HIVE: ', returned ) jobInfo = jobDB.getJobAttributes( self.__jobID ) if not jobInfo['OK']: return S_ERROR( jobInfo['Value'] ) proxy = "" jobInfo = jobInfo['Value'] if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ): proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ) else: proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ) HiveJob = "InputSandbox" + str( self.__jobID ) + "/" + executableFile HiveJobOutput = str( self.__jobID ) + "_" + executableFile + "_out" returned = HiveV1Cli.jobSubmit( tempPath, HiveJob, proxy['chain'], HiveJobOutput ) self.log.info( 'Launch Hadoop-Hive job to the Master: ', returned ) if not returned['OK']: return S_ERROR( returned['Message'] ) else: self.log.info( 'Hadoop-Hive Job ID: ', returned['Value'] ) return S_OK( returned['Value'] )
def _getPilotProxyFromDIRACGroup( self, ownerDN, ownerGroup, requiredTimeLeft ): """ Download a limited pilot proxy with VOMS extensions depending on the group """ if gProxyManager.userHasProxy( ownerDN, ownerGroup ): return gProxyManager.downloadProxyToFile( ownerDN, ownerGroup ) #Assign VOMS attribute vomsAttr = CS.getVOMSAttributeForGroup( ownerGroup ) if not vomsAttr: self.log.info( "Downloading a proxy without VOMS extensions for %s@%s" % ( ownerDN, ownerGroup ) ) return gProxyManager.downloadProxy( ownerDN, ownerGroup, limited = True, requiredTimeLeft = requiredTimeLeft ) else: self.log.info( "Downloading a proxy with '%s' VOMS extension for %s@%s" % ( vomsAttr, ownerDN, ownerGroup ) ) return gProxyManager.downloadVOMSProxy( ownerDN, ownerGroup, limited = True, requiredTimeLeft = requiredTimeLeft, requiredVOMSAttribute = vomsAttr )
def submitNewBigPilot(self): tempPath = self.__tmpSandBoxDir + str(self.__jobID) dirac = Dirac() if not os.path.exists(tempPath): os.makedirs(tempPath) settingJobSandBoxDir = dirac.getInputSandbox(self.__jobID, tempPath) self.log.info("Writting temporal SandboxDir in Server", settingJobSandBoxDir) jobXMLName = "job:" + str(self.__jobID) + ".xml" with open(os.path.join(tempPath, jobXMLName), "wb") as temp_file: temp_file.write(self.jobWrapper()) self.log.info("Writting temporal Hadoop Job.xml") HadoopV1cli = HadoopV1Client(self.__User, self.__publicIP, self.__Port) # returned = HadoopV1cli.dataCopy( tempPath, self.__tmpSandBoxDir ) # self.log.info( 'Copy the job contain to the Hadoop Master: ', returned ) jobInfo = jobDB.getJobAttributes(self.__jobID) if not jobInfo["OK"]: return S_ERROR(jobInfo["Value"]) proxy = "" jobInfo = jobInfo["Value"] if gProxyManager.userHasProxy(jobInfo["OwnerDN"], jobInfo["OwnerGroup"]): proxy = gProxyManager.downloadProxyToFile(jobInfo["OwnerDN"], jobInfo["OwnerGroup"]) else: proxy = self.__requestProxyFromProxyManager(jobInfo["OwnerDN"], jobInfo["OwnerGroup"]) returned = HadoopV1cli.submitPilotJob(tempPath, jobXMLName, proxy["chain"]) self.log.info("Launch Hadoop pilot to the Hadoop Master: ", returned) if not returned["OK"]: return S_ERROR(returned["Value"]) else: self.log.info("Hadoop Job ID: ", returned["Value"]) return S_OK(returned["Value"])
def submitNewBigJob( self ): tempPath = self.__tmpSandBoxDir + str( self.__jobID ) dirac = Dirac() if not os.path.exists( tempPath ): os.makedirs( tempPath ) settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath ) self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir ) jobXMLName = "job:" + str( self.__jobID ) + '.xml' with open( os.path.join( tempPath, jobXMLName ), 'wb' ) as temp_file: temp_file.write( self.jobWrapper() ) self.log.info( 'Writting temporal Hadoop Job.xml' ) HadoopV1cli = HadoopV2Client( self.__User , self.__publicIP ) returned = HadoopV1cli.dataCopy( tempPath, self.__tmpSandBoxDir ) self.log.info( 'Copy the job contain to the Hadoop Master: ', returned ) jobInfo = jobDB.getJobAttributes( self.__jobID ) if not jobInfo['OK']: return S_ERROR( jobInfo['Value'] ) proxy = "" jobInfo = jobInfo['Value'] if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ): proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ) else: proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ) returned = HadoopV1cli.jobSubmit( tempPath, jobXMLName, proxy['chain'] ) self.log.info( 'Launch Hadoop job to the Hadoop Master: ', returned ) if not returned['OK']: return S_ERROR( returned['Message'] ) else: self.log.info( 'Hadoop Job ID: ', returned['Value'] ) return S_OK( returned['Value'] )
def submitNewBigJob( self ): #1.- Creamos carpeta temporal self.log.debug( 'Step1::: mkdir temp folder' ) tempPath = self.__tmpSandBoxDir + str( self.__jobID ) + "/" dirac = Dirac() if not os.path.exists( tempPath ): os.makedirs( tempPath ) #2.- Introducimos el contenido del inputsandbox en la carpeta temporal self.log.debug( 'Step2::: download inputsand to temp folder' ) settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath ) self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir ) moveData = tempPath + "/InputSandbox" + str( self.__jobID ) #3.- Move the data to client self.log.debug( 'Step2::: download inputsandbox to temp folder' ) HadoopV1InteractiveCli = HadoopV1InteractiveClient( self.__User , self.__publicIP, self.__Port ) returned = HadoopV1InteractiveCli.dataCopy( tempPath, self.__tmpSandBoxDir ) self.log.debug( 'Returned of copy the job contain to the Hadoop Master with HadoopInteractive::: ', returned ) #3.- Get executable file result = jobDB.getJobJDL( str( self.__jobID ) , True ) classAdJob = ClassAd( result['Value'] ) executableFile = "" if classAdJob.lookupAttribute( 'Executable' ): executableFile = classAdJob.getAttributeString( 'Executable' ) self.log.debug( 'Step3::: Get executable file: ', executableFile ) jobInfo = jobDB.getJobAttributes( self.__jobID ) if not jobInfo['OK']: return S_ERROR( jobInfo['Value'] ) proxy = "" jobInfo = jobInfo['Value'] if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ): proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ) else: proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ) HadoopInteractiveJob = "InputSandbox" + str( self.__jobID ) + "/" + executableFile HadoopInteractiveJobCommand = "InputSandbox" + str( self.__jobID ) + "/" + executableFile + " " + self.__JobName HadoopInteractiveJobOutput = tempPath + str( self.__jobID ) + "_" + executableFile + "_out" #4.- Creating second part of the job name if ( len( re.split( " ", self.__JobName ) ) > 1 ): #(name for random writter with -D)name_job = re.split( " ", self.__JobName )[0] + " " + re.split( " ", self.__JobName )[1] + " " + re.split( " ", self.__JobName )[2] name_job = re.split( " ", self.__JobName )[0] + " " + re.split( " ", self.__JobName )[1] #(name for random writter with -D)output_job = moveData + "/" + re.split( " ", self.__JobName )[3] #(name for random writter with -D)cfg_job = "" #(name for random writter with -D)if ( len( re.split( " ", self.__JobName ) ) > 4 ): #(name for random writter with -D) cfg_job = moveData + "/" + re.split( " ", self.__JobName )[4] #5.- Parsing execution command #cmd = "hadoop jar " + tempPath + HadoopInteractiveJob + " " + name_job + " " + output_job + " " + cfg_job cmd = "hadoop jar " + tempPath + HadoopInteractiveJob + " " + name_job + " " + tempPath + "/InputSandbox" + str( self.__jobID ) + "/" + "/dataset-USC-a-grep '[and]+'" else: dataset = re.split( "/", self.__Dataset ) count = 0 datasetname = "" for dir in dataset: count = count + 1 if ( count > 2 ): datasetname = datasetname + "/" + dir cmd = "hadoop jar " + tempPath + HadoopInteractiveJob + " " + self.__JobName + " " + datasetname + " " + tempPath + "/" + self.__JobName.replace( " ", "" ) + "_" + str( self.__jobID ) self.log.debug( 'Step4::: Making CMD for submission: ', cmd ) self.log.debug( 'Step5::: Submit file to hadoop: ' ) returned = HadoopV1InteractiveCli.jobSubmit( tempPath, HadoopInteractiveJob, proxy['chain'], HadoopInteractiveJobOutput, cmd ) self.log.info( 'Launch Hadoop-HadoopInteractive job to the Master: ', returned ) if not returned['OK']: return S_ERROR( returned['Message'] ) else: self.log.info( 'Hadoop-HadoopInteractive Job ID: ', returned['Value'] ) return S_OK( returned['Value'] )