def web_getSelectionData(self):
sData = self.getSessionData()
callback = {}
user = sData["user"]["username"]
if user == "Anonymous":
self.finish({
"success": "false",
"error": "You are not authorize to access these data"
})
if len(self.request.arguments) > 0:
tmp = {}
for i in self.request.arguments:
tmp[i] = str(self.request.arguments[i])
callback["extra"] = tmp
rpcClient = RPCClient("Framework/ProxyManager")
retVal = yield self.threadTask(rpcClient.getContents, {}, [], 0, 0)
if not retVal["OK"]:
self.finish({"success": "false", "error": retVal["Message"]})
data = retVal["Value"]
users = []
groups = []
for record in data["Records"]:
users.append(str(record[0]))
groups.append(str(record[2]))
users = uniqueElements(users)
groups = uniqueElements(groups)
users.sort()
groups.sort()
users = map(lambda x: [x], users)
groups = map(lambda x: [x], groups)
callback["username"] = users
callback["usergroup"] = groups
result = gConfig.getOption(
"/WebApp/ProxyManagementMonitoring/TimeSpan",
"86400,432000,604800,2592000")
if result["OK"]:
tmp = result["Value"]
tmp = tmp.split(", ")
if len(tmp) > 0:
timespan = []
for i in tmp:
human_readable = self.__humanize_time(i)
timespan.append([i, human_readable])
else:
timespan = [["Nothing to display"]]
else:
timespan = [["Error during RPC call"]]
callback["expiredBefore"] = timespan
callback["expiredAfter"] = timespan
self.finish(callback)
def web_getSelectionData( self ):
sData = self.getSessionData()
callback = {}
user = sData["user"]["username"]
if user == "Anonymous":
self.finish( {"success":"false", "error":"You are not authorize to access these data"} )
if len( self.request.arguments ) > 0:
tmp = {}
for i in self.request.arguments:
tmp[i] = str( self.request.arguments[i] )
callback["extra"] = tmp
rpcClient = RPCClient( "Framework/ProxyManager" )
retVal = yield self.threadTask( rpcClient.getContents, {}, [], 0, 0 )
if not retVal[ "OK" ]:
self.finish( {"success":"false", "error":retVal["Message"]} )
data = retVal[ "Value" ]
users = []
groups = []
for record in data[ "Records" ]:
users.append( str( record[0] ) )
groups.append( str( record[2] ) )
users = uniqueElements( users )
groups = uniqueElements( groups )
users.sort()
groups.sort()
users = map( lambda x: [x], users )
groups = map( lambda x: [x], groups )
callback["username"] = users
callback["usergroup"] = groups
result = gConfig.getOption( "/WebApp/ProxyManagementMonitoring/TimeSpan", "86400,432000,604800,2592000" )
if result["OK"]:
tmp = result["Value"]
tmp = tmp.split( ", " )
if len( tmp ) > 0:
timespan = []
for i in tmp:
human_readable = self.__humanize_time( i )
timespan.append( [i, human_readable] )
else:
timespan = [["Nothing to display"]]
else:
timespan = [["Error during RPC call"]]
callback["expiredBefore"] = timespan
callback["expiredAfter"] = timespan
self.finish( callback )
def _prepareRemoteHost(self, host=None ):
""" Prepare remote directories and upload control script
"""
ssh = SSH( host = host, parameters = self.ceParameters )
# Make remote directories
dirTuple = tuple ( uniqueElements( [self.sharedArea,
self.executableArea,
self.infoArea,
self.batchOutput,
self.batchError,
self.workArea] ) )
nDirs = len( dirTuple )
cmd = 'mkdir -p %s; '*nDirs % dirTuple
self.log.verbose( 'Creating working directories on %s' % self.ceParameters['SSHHost'] )
result = ssh.sshCall( 30, cmd )
if not result['OK']:
self.log.warn( 'Failed creating working directories: %s' % result['Message'][1] )
return result
status,output,error = result['Value']
if status == -1:
self.log.warn( 'TImeout while creating directories' )
return S_ERROR( 'TImeout while creating directories' )
if "cannot" in output:
self.log.warn( 'Failed to create directories: %s' % output )
return S_ERROR( 'Failed to create directories: %s' % output )
# Upload the control script now
sshScript = os.path.join( rootPath, "DIRAC", "Resources", "Computing", "remote_scripts", self.controlScript )
self.log.verbose( 'Uploading %s script to %s' % ( self.controlScript, self.ceParameters['SSHHost'] ) )
result = ssh.scpCall( 30, sshScript, self.sharedArea )
if not result['OK']:
self.log.warn( 'Failed uploading control script: %s' % result['Message'][1] )
return result
status,output,error = result['Value']
if status != 0:
if status == -1:
self.log.warn( 'Timeout while uploading control script' )
return S_ERROR( 'Timeout while uploading control script' )
else:
self.log.warn( 'Failed uploading control script: %s' % output )
return S_ERROR( 'Failed uploading control script' )
# Chmod the control scripts
self.log.verbose( 'Chmod +x control script' )
result = ssh.sshCall( 10, "chmod +x %s/%s" % ( self.sharedArea, self.controlScript ) )
if not result['OK']:
self.log.warn( 'Failed chmod control script: %s' % result['Message'][1] )
return result
status,output,error = result['Value']
if status != 0:
if status == -1:
self.log.warn( 'Timeout while chmod control script' )
return S_ERROR( 'Timeout while chmod control script' )
else:
self.log.warn( 'Failed uploading chmod script: %s' % output )
return S_ERROR( 'Failed uploading chmod script' )
return S_OK()
def getCompatiblePlatforms( originalPlatforms ):
""" Get a list of platforms compatible with the given list
"""
if type( originalPlatforms ) == type( ' ' ):
platforms = [originalPlatforms]
else:
platforms = list( originalPlatforms )
platforms = list( platform.replace( ' ', '' ) for platform in platforms )
result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' )
if not ( result['OK'] and result['Value'] ):
return S_ERROR( "OS compatibility info not found" )
platformsDict = dict( [( k, v.replace( ' ', '' ).split( ',' ) ) for k, v in result['Value'].iteritems()] )
for k, v in platformsDict.iteritems():
if k not in v:
v.append( k )
resultList = list( platforms )
for p in platforms:
tmpList = platformsDict.get( p, [] )
for pp in platformsDict:
if p in platformsDict[pp]:
tmpList.append( pp )
tmpList += platformsDict[pp]
if tmpList:
resultList += tmpList
return S_OK( uniqueElements( resultList ) )
def _prepareHost(self):
""" Prepare directories and copy control script
"""
# Make remote directories
dirTuple = uniqueElements([
self.sharedArea, self.executableArea, self.infoArea,
self.batchOutput, self.batchError, self.workArea
])
nDirs = len(dirTuple)
cmdTuple = ['mkdir', '-p'] + dirTuple
self.log.verbose('Creating working directories')
result = systemCall(30, cmdTuple)
if not result['OK']:
self.log.warn('Failed creating working directories: %s' %
result['Message'][1])
return result
status, output, error = result['Value']
if status != 0:
self.log.warn('Failed to create directories: %s' % output)
return S_ERROR('Failed to create directories: %s' % output)
# copy the control script now
localScript = os.path.join(rootPath, "DIRAC", "Resources", "Computing",
"remote_scripts", self.controlScript)
self.log.verbose('Copying %s script' % self.controlScript)
try:
shutil.copy(localScript, self.sharedArea)
# Chmod the control scripts
self.finalScript = os.path.join(self.sharedArea,
self.controlScript)
os.chmod(self.finalScript, 0o755)
except Exception, x:
self.log.warn('Failed copying control script', x)
return S_ERROR(x)
def getUserByProperty( self , prop = "NormalUser" ):
"""
Get usernames based on group property
Argument is a string. Return value is a list
"""
groupList = list()
result = gConfig.getSections( "/Registry/Groups" )
gLogger.debug( "Group response: %s" % result )
if not result[ "OK" ]:
return groupList
groups = result[ "Value" ]
for j in groups:
props = getProperties( j )
gLogger.debug( "%s properties: %s" % ( j , props ) )
if prop in props:
groupList.append( j )
if not len( groupList ) > 0:
return groupList
groupList = uniqueElements( groupList )
gLogger.debug( "Chosen group(s): %s" % groupList )
userList = list()
for i in groupList:
users = gConfig.getValue( "/Registry/Groups/%s/Users" % i , [] )
gLogger.debug( "%s users: %s" % ( i , users ) )
if len( users ) > 0:
userList.extend( users )
return userList
def _prepareRemoteHost(self, host=None ):
""" Prepare remote directories and upload control script
"""
ssh = SSH( host = host, parameters = self.ceParameters )
# Make remote directories
dirTuple = tuple ( uniqueElements( [self.sharedArea,
self.executableArea,
self.infoArea,
self.batchOutput,
self.batchError,
self.workArea] ) )
nDirs = len( dirTuple )
cmd = 'mkdir -p %s; '*nDirs % dirTuple
self.log.verbose( 'Creating working directories on %s' % self.ceParameters['SSHHost'] )
result = ssh.sshCall( 30, cmd )
if not result['OK']:
self.log.warn( 'Failed creating working directories: %s' % result['Message'][1] )
return result
status,output,error = result['Value']
if status == -1:
self.log.warn( 'TImeout while creating directories' )
return S_ERROR( 'TImeout while creating directories' )
if "cannot" in output:
self.log.warn( 'Failed to create directories: %s' % output )
return S_ERROR( 'Failed to create directories: %s' % output )
# Upload the control script now
sshScript = os.path.join( rootPath, "DIRAC", "Resources", "Computing", "remote_scripts", self.controlScript )
self.log.verbose( 'Uploading %s script to %s' % ( self.controlScript, self.ceParameters['SSHHost'] ) )
result = ssh.scpCall( 30, sshScript, self.sharedArea )
if not result['OK']:
self.log.warn( 'Failed uploading control script: %s' % result['Message'][1] )
return result
status,output,error = result['Value']
if status != 0:
if status == -1:
self.log.warn( 'Timeout while uploading control script' )
return S_ERROR( 'Timeout while uploading control script' )
else:
self.log.warn( 'Failed uploading control script: %s' % output )
return S_ERROR( 'Failed uploading control script' )
# Chmod the control scripts
self.log.verbose( 'Chmod +x control script' )
result = ssh.sshCall( 10, "chmod +x %s/%s" % ( self.sharedArea, self.controlScript ) )
if not result['OK']:
self.log.warn( 'Failed chmod control script: %s' % result['Message'][1] )
return result
status,output,error = result['Value']
if status != 0:
if status == -1:
self.log.warn( 'Timeout while chmod control script' )
return S_ERROR( 'Timeout while chmod control script' )
else:
self.log.warn( 'Failed uploading chmod script: %s' % output )
return S_ERROR( 'Failed uploading chmod script' )
return S_OK()
def getUserByProperty(self, prop="NormalUser"):
"""
Get usernames based on group property
Argument is a string. Return value is a list
"""
groupList = list()
result = gConfig.getSections("/Registry/Groups")
gLogger.debug("Group response: %s" % result)
if not result["OK"]:
return groupList
groups = result["Value"]
for j in groups:
props = getProperties(j)
gLogger.debug("%s properties: %s" % (j, props))
if prop in props:
groupList.append(j)
if not len(groupList) > 0:
return groupList
groupList = uniqueElements(groupList)
gLogger.debug("Chosen group(s): %s" % groupList)
userList = list()
for i in groupList:
users = gConfig.getValue("/Registry/Groups/%s/Users" % i, [])
gLogger.debug("%s users: %s" % (i, users))
if len(users) > 0:
userList.extend(users)
return userList
def getCompatiblePlatforms( originalPlatforms ):
""" Get a list of platforms compatible with the given list
"""
if type( originalPlatforms ) == type( ' ' ):
platforms = [originalPlatforms]
else:
platforms = list( originalPlatforms )
platformDict = {}
result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' )
if result['OK'] and result['Value']:
platformDict = result['Value']
for platform in platformDict:
platformDict[platform] = [ x.strip() for x in platformDict[platform].split( ',' ) ]
else:
return S_ERROR( 'OS compatibility info not found' )
resultList = list( platforms )
for p in platforms:
tmpList = platformDict.get( p, [] )
for pp in platformDict:
if p in platformDict[pp]:
tmpList.append( pp )
tmpList += platformDict[pp]
if tmpList:
resultList += tmpList
return S_OK( uniqueElements( resultList ) )
def getCompatiblePlatforms(originalPlatforms):
""" Get a list of platforms compatible with the given list
"""
if type(originalPlatforms) == type(' '):
platforms = [originalPlatforms]
else:
platforms = list(originalPlatforms)
platformDict = {}
result = gConfig.getOptionsDict('/Resources/Computing/OSCompatibility')
if result['OK'] and result['Value']:
platformDict = result['Value']
for platform in platformDict:
platformDict[platform] = [
x.strip() for x in platformDict[platform].split(',')
]
else:
return S_ERROR('OS compatibility info not found')
resultList = list(platforms)
for p in platforms:
tmpList = platformDict.get(p, [])
for pp in platformDict:
if p in platformDict[pp]:
tmpList.append(pp)
tmpList += platformDict[pp]
if tmpList:
resultList += tmpList
return S_OK(uniqueElements(resultList))
def _prepareHost( self ):
""" Prepare directories and copy control script
"""
# Make remote directories
dirTuple = uniqueElements( [ self.sharedArea,
self.executableArea,
self.infoArea,
self.batchOutput,
self.batchError,
self.workArea] )
nDirs = len( dirTuple )
cmdTuple = [ 'mkdir', '-p' ] + dirTuple
self.log.verbose( 'Creating working directories' )
result = systemCall( 30, cmdTuple )
if not result['OK']:
self.log.warn( 'Failed creating working directories: %s' % result['Message'][1] )
return result
status, output, error = result['Value']
if status != 0:
self.log.warn( 'Failed to create directories: %s' % output )
return S_ERROR( 'Failed to create directories: %s' % output )
# copy the control script now
localScript = os.path.join( rootPath, "DIRAC", "Resources", "Computing", "remote_scripts", self.controlScript )
self.log.verbose( 'Copying %s script' % self.controlScript )
try:
shutil.copy( localScript, self.sharedArea )
# Chmod the control scripts
self.finalScript = os.path.join( self.sharedArea, self.controlScript )
os.chmod( self.finalScript, 0o755 )
except Exception, x:
self.log.warn( 'Failed copying control script', x )
return S_ERROR( x )
def getCompatiblePlatforms(originalPlatforms):
""" Get a list of platforms compatible with the given list
"""
if isinstance(originalPlatforms, six.string_types):
platforms = [originalPlatforms]
else:
platforms = list(originalPlatforms)
platforms = list(platform.replace(' ', '') for platform in platforms)
result = gConfig.getOptionsDict('/Resources/Computing/OSCompatibility')
if not (result['OK'] and result['Value']):
return S_ERROR("OS compatibility info not found")
platformsDict = dict(
(k, v.replace(' ', '').split(','))
for k, v in result['Value'].items()) # can be an iterator
for k, v in platformsDict.items(): # can be an iterator
if k not in v:
v.append(k)
resultList = list(platforms)
for p in platforms:
tmpList = platformsDict.get(p, [])
for pp in platformsDict:
if p in platformsDict[pp]:
tmpList.append(pp)
tmpList += platformsDict[pp]
if tmpList:
resultList += tmpList
return S_OK(uniqueElements(resultList))
def __getSelectionData(self):
callback = {}
if not authorizeAction():
return {"success":"false","error":"You are not authorize to access these data"}
if len(request.params) > 0:
tmp = {}
for i in request.params:
tmp[i] = str(request.params[i])
callback["extra"] = tmp
rpcClient = getRPCClient( "Framework/ProxyManager" )
retVal = rpcClient.getContents( {}, [], 0, 0 )
if not retVal[ "OK" ]:
return {"success":"false","error":retVal["Message"]}
data = retVal[ "Value" ]
users = []
groups = []
for record in data[ "Records" ]:
users.append( str(record[0]) )
groups.append( str(record[2]) )
users = uniqueElements(users)
groups = uniqueElements(groups)
users.sort()
groups.sort()
users = map(lambda x: [x], users)
groups = map(lambda x: [x], groups)
if len(users) > 1:
users.insert(0,["All"])
if len(groups) > 1:
groups.insert(0,["All"])
callback["username"] = users
callback["usergroup"] = groups
result = gConfig.getOption("/Website/ProxyManagementMonitoring/TimeSpan")
if result["OK"]:
tmp = result["Value"]
tmp = tmp.split(", ")
if len(tmp)>0:
timespan = []
for i in tmp:
human_readable = self.__humanize_time(i)
timespan.append([i, human_readable])
else:
timespan = [["Nothing to display"]]
else:
timespan = [["Error during RPC call"]]
callback["expiredBefore"] = timespan
callback["expiredAfter"] = timespan
return callback
def _getFileGroups( self, fileReplicas ):
fileGroups = {}
for lfn, replicas in fileReplicas.items():
replicaSEs = str.join( ',', sortList( uniqueElements( replicas.keys() ) ) )
if not fileGroups.has_key( replicaSEs ):
fileGroups[replicaSEs] = []
fileGroups[replicaSEs].append( lfn )
return fileGroups
def web_sendMessage(self):
"""
Send message(not implemented yet) or email getting parameters from request
"""
email = self.getUserEmail()
if not "subject" in self.request.arguments:
result = "subject parameter is not in request... aborting"
gLogger.debug(result)
self.write({ "success" : "false" , "error" : result })
return
subject = self.checkUnicode(self.request.arguments[ "subject" ][0])
if not len(subject) > 0:
subject = "Message from %s" % email
if not "message" in self.request.arguments:
result = "msg parameter is not in request... aborting"
gLogger.debug(result)
self.write({ "success" : "false" , "error" : result })
return
body = self.checkUnicode(self.request.arguments[ "message" ][0])
if not len(body) > 0:
result = "Message body has zero length... aborting"
gLogger.debug(result)
self.write({ "success" : "false" , "error" : result })
return
users = self.request.arguments[ "users" ][0].split(",")
groups = self.request.arguments[ "groups" ][0].split(",")
gLogger.info("List of groups from request: %s" % groups)
if groups:
for g in groups:
userList = self.getUsersFromGroup(g)
gLogger.info("Get users: %s from group %s" % (userList , g))
if userList:
users.extend(userList)
gLogger.info("Merged list of users from users and group %s" % users)
if not len(users) > 0:
error = "Length of list of recipients is zero size"
gLogger.info(error)
self.write({ "success" : "false" , "error" : error })
return
users = uniqueElements(users)
gLogger.info("Final list of users to send message/mail: %s" % users)
sendDict = self.getMailDict(users)
self.write(self.sendMail(sendDict , subject , body , email))
def __sendMessage( self ):
"""
Send message(not implemented yet) or email getting parameters from request
"""
getEmail = self.__returnEmail()
if not "result" in getEmail:
return getEmail
email = getEmail[ "result" ]
if not "subject" in request.params:
result = "subject parameter is not in request... aborting"
gLogger.debug( result )
return { "success" : "false" , "error" : result }
subject = GeneralController().checkUnicode( request.params[ "subject" ] )
if not len( subject ) > 0:
subject = "Message from %s" % email
if not "msg" in request.params:
result = "msg parameter is not in request... aborting"
gLogger.debug( result )
return { "success" : "false" , "error" : result }
body = GeneralController().checkUnicode( request.params[ "msg" ] )
if not len( body ) > 0:
result = "Message body has zero length... aborting"
gLogger.debug( result )
return { "success" : "false" , "error" : result }
users = list()
userList = GeneralController().userlistFromRequest()
gLogger.info( "List of users from request: %s" % userList )
if userList:
users.extend( userList )
groupList = GeneralController().grouplistFromRequest()
gLogger.info( "List of groups from request: %s" % groupList )
if groupList:
for i in groupList:
userList = GeneralController().userlistFromGroup( i )
gLogger.info( "Get users: %s from group %s" % ( userList , i ) )
if userList:
users.extend( userList )
gLogger.info( "Merged list of users from users and group %s" % users )
if not len( users ) > 0:
error = "Length of list of recipients is zero size"
gLogger.info( error )
return { "success" : "false" , "error" : error }
users = uniqueElements( users )
gLogger.info( "Final list of users to send message/mail: %s" % users )
if "email" in request.params:
sendDict = GeneralController().getMailDict( users )
return GeneralController().sendMail( sendDict , subject , body , email )
return { "success" : "false" , "error" : result }
def web_sendMessage(self):
"""
Send message(not implemented yet) or email getting parameters from request
"""
email = self.getUserEmail()
if not "subject" in self.request.arguments:
result = "subject parameter is not in request... aborting"
gLogger.debug(result)
self.write({"success": "false", "error": result})
return
subject = self.checkUnicode(self.request.arguments["subject"][0])
if not subject:
subject = "Message from %s" % email
if not "message" in self.request.arguments:
result = "msg parameter is not in request... aborting"
gLogger.debug(result)
self.write({"success": "false", "error": result})
return
body = self.checkUnicode(self.request.arguments["message"][0])
if not len(body) > 0:
result = "Message body has zero length... aborting"
gLogger.debug(result)
self.write({"success": "false", "error": result})
return
users = self.request.arguments["users"][0].split(",")
groups = self.request.arguments["groups"][0].split(",")
gLogger.info("List of groups from request: %s" % groups)
if groups:
for g in groups:
userList = self.getUsersFromGroup(g)
gLogger.info("Get users: %s from group %s" % (userList, g))
if userList:
users.extend(userList)
gLogger.info("Merged list of users from users and group %s" % users)
if not len(users) > 0:
error = "Length of list of recipients is zero size"
gLogger.info(error)
self.write({"success": "false", "error": error})
return
users = uniqueElements(users)
gLogger.info("Final list of users to send message/mail: %s" % users)
sendDict = self.getMailDict(users)
self.write(self.sendMail(sendDict, subject, body, email))
def _getFileGroups(cls, fileReplicas):
""" get file groups dictionary { "SE1,SE2,SE3" : [ lfn1, lfn2 ], ... }
:param dict fileReplicas: { lfn : [SE1, SE2, SE3], ... }
"""
fileGroups = {}
for lfn, replicas in fileReplicas.items():
replicaSEs = ",".join(sortList(uniqueElements(replicas)))
if replicaSEs not in fileGroups:
fileGroups[replicaSEs] = []
fileGroups[replicaSEs].append(lfn)
return fileGroups
def _getFileGroups( cls, fileReplicas ):
""" get file groups dictionary { "SE1,SE2,SE3" : [ lfn1, lfn2 ], ... }
:param dict fileReplicas: { lfn : [SE1, SE2, SE3], ... }
"""
fileGroups = {}
for lfn, replicas in fileReplicas.items():
replicaSEs = ",".join( sortList( uniqueElements( replicas ) ) )
if replicaSEs not in fileGroups:
fileGroups[replicaSEs] = []
fileGroups[replicaSEs].append( lfn )
return fileGroups
def __getLaunchpadOpts(self):
gLogger.info( "start __getLaunchpadOpts" )
delimiter = gConfig.getValue( "/Website/Launchpad/ListSeparator" , ',' )
options = self.__getOptionsFromCS( delimiter = delimiter)
platform = self.__getPlatform()
if platform and options:
if not options.has_key( "Platform" ):
options[ "Platform" ] = platform
else:
csPlatform = list( options[ "Platform" ] )
allPlatforms = csPlatform + platform
platform = uniqueElements( allPlatforms )
options[ "Platform" ] = platform
gLogger.debug( "Combined options from CS: %s" % options )
override = gConfig.getValue( "/Website/Launchpad/OptionsOverride" , False)
gLogger.info( "end __getLaunchpadOpts" )
return {"success":"true","result":options,"override":override,"separator":delimiter}
def __getLayout( self ) :
gLogger.info( "Running getLayout()" )
msg = "getLayout() for %s@%s" % ( getUsername() , getSelectedGroup() )
upc = UserProfileClient( USER_PROFILE_NAME, getRPCClient )
result = upc.listAvailableVars()
gLogger.debug( result )
if not result[ "OK" ]:
gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return { "success" : "false" , "error" : result[ "Message" ] }
result = result[ "Value" ]
gLogger.always( "array2obj" )
availble = map( self.__array2obj , result )
gLogger.always( availble )
users = list()
for i in result :
if len( i ) > 1 :
users.append( { "user" : i[ 0 ] } )
users = uniqueElements( users )
gLogger.info( "Result %s: %s AND %s" % ( msg , availble , users ) )
return { "success" : "true" , "result" : availble , "users" : users }
def __getLayout(self):
gLogger.info("Running getLayout()")
msg = "getLayout() for %s@%s" % (getUsername(), getSelectedGroup())
upc = UserProfileClient(USER_PROFILE_NAME, getRPCClient)
result = upc.listAvailableVars()
gLogger.debug(result)
if not result["OK"]:
gLogger.error("Result %s: %s" % (msg, result["Message"]))
return {"success": "false", "error": result["Message"]}
result = result["Value"]
gLogger.always("array2obj")
availble = map(self.__array2obj, result)
gLogger.always(availble)
users = list()
for i in result:
if len(i) > 1:
users.append({"user": i[0]})
users = uniqueElements(users)
gLogger.info("Result %s: %s AND %s" % (msg, availble, users))
return {"success": "true", "result": availble, "users": users}
def layoutUser(self):
upProfileName = "Summary"
upc = UserProfileClient( "Summary", getRPCClient )
result = upc.listAvailableVars()
if result["OK"]:
result = result["Value"]
userList = []
for i in result:
userList.append(i[0])
userList = uniqueElements(userList)
resultList = []
for j in userList:
resultList.append({'name':j})
total = len(resultList)
resultList.sort()
resultList.insert(0,{'name':'All'})
c.result = {"success":"true","result":resultList,"total":total}
else:
c.result = {"success":"false","error":result["Message"]}
return c.result
def _prepareHost(self):
"""Prepare directories and copy control script"""
# Make remote directories
dirTuple = uniqueElements([
self.sharedArea, self.executableArea, self.infoArea,
self.batchOutput, self.batchError, self.workArea
])
cmdTuple = ["mkdir", "-p"] + dirTuple
self.log.verbose("Creating working directories")
result = systemCall(30, cmdTuple)
if not result["OK"]:
self.log.error("Failed creating working directories",
"(%s)" % result["Message"][1])
return result
status, output, error = result["Value"]
if status != 0:
self.log.error("Failed to create directories", "(%s)" % error)
return S_ERROR(errno.EACCES, "Failed to create directories")
return S_OK()
def __setHistory( self , item , state ):
"""
Insert item to Load or Save history list in first position and checking for
duplications.
Return resulting list
"item" is a dict
"state" should be either "Save" or "Load" but can be any other value
"""
gLogger.info( "Running setHistory( %s , %s )" % ( item , state ) )
msg = "setHistory() for %s@%s" % ( getUsername() , getSelectedGroup() )
opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory"
history_length = gConfig.getOptions( opt , 5 )
upc = UserProfileClient( "Default" , getRPCClient )
group = str( getSelectedGroup() )
profile_name = USER_PROFILE_NAME + ".History." + state + "." + group
result = upc.retrieveVar( profile_name )
gLogger.info( result )
if not result[ "OK" ]:
if result[ "Message" ].find( "No data" ) < 0 :
gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return S_ERROR( result[ "Message" ] )
history = list()
else:
history = result[ "Value" ]
if not isinstance( history , list ):
err = "List expected at: %s" % profile_name
gLogger.error( "Result %s: %s" % ( msg , err ) )
return S_ERROR( err )
if( len( history ) > history_length ):
history = result[ history_length ]
history.insert( 0 , item )
history = uniqueElements( history )
gLogger.error( "History: %s" % history )
result = upc.storeVar( profile_name , history )
gLogger.info( result )
if not result[ "OK" ]:
gLogger.error( "Result %s: %s" % ( msg , result[ "Message" ] ) )
return S_ERROR( result[ "Message" ] )
gLogger.info( "Result %s: %s" % ( msg , history ) )
return S_OK( history )
def __setHistory(self, item, state):
"""
Insert item to Load or Save history list in first position and checking for
duplications.
Return resulting list
"item" is a dict
"state" should be either "Save" or "Load" but can be any other value
"""
gLogger.info("Running setHistory( %s , %s )" % (item, state))
msg = "setHistory() for %s@%s" % (getUsername(), getSelectedGroup())
opt = "/Website/" + USER_PROFILE_NAME + "/ShowHistory"
history_length = gConfig.getOptions(opt, 5)
upc = UserProfileClient("Default", getRPCClient)
group = str(getSelectedGroup())
profile_name = USER_PROFILE_NAME + ".History." + state + "." + group
result = upc.retrieveVar(profile_name)
gLogger.info(result)
if not result["OK"]:
if result["Message"].find("No data") < 0:
gLogger.error("Result %s: %s" % (msg, result["Message"]))
return S_ERROR(result["Message"])
history = list()
else:
history = result["Value"]
if not isinstance(history, list):
err = "List expected at: %s" % profile_name
gLogger.error("Result %s: %s" % (msg, err))
return S_ERROR(err)
if (len(history) > history_length):
history = result[history_length]
history.insert(0, item)
history = uniqueElements(history)
gLogger.error("History: %s" % history)
result = upc.storeVar(profile_name, history)
gLogger.info(result)
if not result["OK"]:
gLogger.error("Result %s: %s" % (msg, result["Message"]))
return S_ERROR(result["Message"])
gLogger.info("Result %s: %s" % (msg, history))
return S_OK(history)
def _getPoolCatalogs(directory=""):
patterns = ["*.xml", "*.xml*gz"]
omissions = [r"\.bak$"] # to be ignored for production files
# First obtain valid list of unpacked catalog files in directory
poolCatalogList = []
for pattern in patterns:
fileList = glob.glob(os.path.join(directory, pattern))
for fname in fileList:
if fname.endswith(".bak"):
gLogger.verbose("Ignoring BAK file: %s" % fname)
elif tarfile.is_tarfile(fname):
gLogger.debug("Unpacking catalog XML file %s" %
(os.path.join(directory, fname)))
with tarfile.open(os.path.join(directory, fname), "r") as tf:
for member in tf.getmembers():
tf.extract(member, directory)
poolCatalogList.append(
os.path.join(directory, member.name))
else:
poolCatalogList.append(fname)
poolCatalogList = uniqueElements(poolCatalogList)
# Now have list of all XML files but some may not be Pool XML catalogs...
finalCatList = []
for possibleCat in poolCatalogList:
try:
_cat = PoolXMLCatalog(possibleCat)
finalCatList.append(possibleCat)
except Exception as x:
gLogger.debug("Ignoring non-POOL catalogue file %s" % possibleCat)
gLogger.debug("Final list of catalog files are: %s" %
", ".join(finalCatList))
return finalCatList
def _prepareHost(self):
""" Prepare directories and copy control script
"""
# Make remote directories
dirTuple = uniqueElements([
self.sharedArea, self.executableArea, self.infoArea,
self.batchOutput, self.batchError, self.workArea
])
cmdTuple = ['mkdir', '-p'] + dirTuple
self.log.verbose('Creating working directories')
result = systemCall(30, cmdTuple)
if not result['OK']:
self.log.warn('Failed creating working directories: %s' %
result['Message'][1])
return result
status, output, _error = result['Value']
if status != 0:
self.log.warn('Failed to create directories: %s' % output)
return S_ERROR('Failed to create directories: %s' % output)
return S_OK()
def _prepareHost( self ):
""" Prepare directories and copy control script
"""
# Make remote directories
dirTuple = uniqueElements( [ self.sharedArea,
self.executableArea,
self.infoArea,
self.batchOutput,
self.batchError,
self.workArea] )
cmdTuple = [ 'mkdir', '-p' ] + dirTuple
self.log.verbose( 'Creating working directories' )
result = systemCall( 30, cmdTuple )
if not result['OK']:
self.log.warn( 'Failed creating working directories: %s' % result['Message'][1] )
return result
status, output, _error = result['Value']
if status != 0:
self.log.warn( 'Failed to create directories: %s' % output )
return S_ERROR( 'Failed to create directories: %s' % output )
return S_OK()
def layoutUser(self):
upProfileName = "Summary"
upc = UserProfileClient("Summary", getRPCClient)
result = upc.listAvailableVars()
if result["OK"]:
result = result["Value"]
userList = []
for i in result:
userList.append(i[0])
userList = uniqueElements(userList)
resultList = []
for j in userList:
resultList.append({'name': j})
total = len(resultList)
resultList.sort()
resultList.insert(0, {'name': 'All'})
c.result = {
"success": "true",
"result": resultList,
"total": total
}
else:
c.result = {"success": "false", "error": result["Message"]}
return c.result
def _getPoolCatalogs(directory=""):
patterns = ["*.xml", "*.xml*gz"]
omissions = ["\.bak$"] # to be ignored for production files
# First obtain valid list of unpacked catalog files in directory
poolCatalogList = []
for pattern in patterns:
fileList = glob.glob(os.path.join(directory, pattern))
for fname in fileList:
if fname.endswith(".bak"):
gLogger.verbose("Ignoring BAK file: %s" % fname)
elif tarfile.is_tarfile(fname):
gLogger.debug("Unpacking catalog XML file %s" % (os.path.join(directory, fname)))
with tarfile.open(os.path.join(directory, fname), "r") as tf:
for member in tf.getmembers():
tf.extract(member, directory)
poolCatalogList.append(os.path.join(directory, member.name))
else:
poolCatalogList.append(fname)
poolCatalogList = uniqueElements(poolCatalogList)
# Now have list of all XML files but some may not be Pool XML catalogs...
finalCatList = []
for possibleCat in poolCatalogList:
try:
_cat = PoolXMLCatalog(possibleCat)
finalCatList.append(possibleCat)
except Exception as x:
gLogger.debug("Ignoring non-POOL catalogue file %s" % possibleCat)
gLogger.debug("Final list of catalog files are: %s" % ", ".join(finalCatList))
return finalCatList
def _prepareRemoteHost(self, host=None):
""" Prepare remote directories and upload control script
"""
ssh = SSH(host=host, parameters=self.ceParameters)
# Make remote directories
dirTuple = tuple(
uniqueElements([
self.sharedArea, self.executableArea, self.infoArea,
self.batchOutput, self.batchError, self.workArea
]))
nDirs = len(dirTuple)
cmd = 'mkdir -p %s; ' * nDirs % dirTuple
cmd = "bash -c '%s'" % cmd
self.log.verbose('Creating working directories on %s' %
self.ceParameters['SSHHost'])
result = ssh.sshCall(30, cmd)
if not result['OK']:
self.log.warn('Failed creating working directories: %s' %
result['Message'][1])
return result
status, output, _error = result['Value']
if status == -1:
self.log.warn('Timeout while creating directories')
return S_ERROR('Timeout while creating directories')
if "cannot" in output:
self.log.warn('Failed to create directories: %s' % output)
return S_ERROR('Failed to create directories: %s' % output)
# Upload the control script now
batchSystemDir = os.path.join(rootPath, "DIRAC", "Resources",
"Computing", "BatchSystems")
batchSystemScript = os.path.join(batchSystemDir,
'%s.py' % self.batchSystem)
batchSystemExecutor = os.path.join(batchSystemDir, 'executeBatch.py')
self.log.verbose('Uploading %s script to %s' %
(self.batchSystem, self.ceParameters['SSHHost']))
remoteScript = '%s/execute_batch' % self.sharedArea
result = ssh.scpCall(30,
'%s %s' %
(batchSystemScript, batchSystemExecutor),
remoteScript,
postUploadCommand='chmod +x %s' % remoteScript)
if not result['OK']:
self.log.warn('Failed uploading control script: %s' %
result['Message'][1])
return result
status, output, _error = result['Value']
if status != 0:
if status == -1:
self.log.warn('Timeout while uploading control script')
return S_ERROR('Timeout while uploading control script')
else:
self.log.warn('Failed uploading control script: %s' % output)
return S_ERROR('Failed uploading control script')
# Chmod the control scripts
#self.log.verbose( 'Chmod +x control script' )
#result = ssh.sshCall( 10, "chmod +x %s/%s" % ( self.sharedArea, self.controlScript ) )
#if not result['OK']:
# self.log.warn( 'Failed chmod control script: %s' % result['Message'][1] )
# return result
#status, output, _error = result['Value']
#if status != 0:
# if status == -1:
# self.log.warn( 'Timeout while chmod control script' )
# return S_ERROR( 'Timeout while chmod control script' )
# else:
# self.log.warn( 'Failed uploading chmod script: %s' % output )
# return S_ERROR( 'Failed uploading chmod script' )
return S_OK()
def registerUser(self,paramcopy):
# Unfortunately there is no way to get rid of empty text values in JS, so i have to hardcode it on server side. Hate it!
default_values = ["John Smith","jsmith","*****@*****.**","+33 9 10 00 10 00","Select prefered virtual organization(s)"]
default_values.append("Select your country")
default_values.append("Any additional information you want to provide to administrators")
dn = getUserDN()
username = getUsername()
if not username == "anonymous":
return {"success":"false","error":"You are already registered in DIRAC with username: %s" % username}
else:
if not dn:
return {"success":"false","error":"You have to load certificate to your browser before trying to register"}
body = ""
userMail = False
vo = []
for i in paramcopy:
if not paramcopy[i] in default_values:
if i == "email":
userMail = paramcopy[i]
if i == "vo":
vo = paramcopy[i].split(",")
body = body + str(i) + ' - "' + str(paramcopy[i]) + '"\n'
if not userMail:
return {"success":"false","error":"Can not get your email from the request"}
gLogger.info("!!! VO: ",vo)
# TODO Check for previous requests
if not len(vo) > 0:
mails = gConfig.getValue("/Website/UserRegistrationEmail",[])
else:
mails = []
for i in vo:
i = i.strip()
voadm = gConfig.getValue("/Registry/VO/%s/VOAdmin" % i,"")
failsafe = False
if voadm:
tmpmail = gConfig.getValue("/Registry/Users/%s/Email" % voadm,"")
if tmpmail:
mails.append(tmpmail)
else:
gLogger.error("Can not find value for option /Registry/Users/%s/Email Trying failsafe option" % voadm)
failsafe = True
else:
gLogger.error("Can not find value for option /Registry/VO/%s/VOAdmin Trying failsafe option" % i)
failsafe = True
if failsafe:
failsafe = gConfig.getValue("/Website/UserRegistrationEmail",[])
if len(failsafe) > 0:
for j in failsafe:
mails.append(j)
else:
gLogger.error("Can not find value for failsafe option /Website/UserRegistrationEmail User registration for VO %s is failed" % i)
mails = uniqueElements(mails)
if not len(mails) > 0:
groupList = list()
allGroups = gConfig.getSections("/Registry/Groups")
if not allGroups["OK"]:
return {"success":"false","error":"No groups found at this DIRAC installation"}
allGroups = allGroups["Value"]
for j in allGroups:
props = getProperties(j)
if "UserAdministrator" in props: # property which usd for user administration
groupList.append(j)
groupList = uniqueElements(groupList)
if not len(groupList) > 0:
return {"success":"false","error":"No groups, resposible for user administration, found"}
userList = list()
for i in groupList:
users = gConfig.getValue("/Registry/Groups/%s/Users" % i,[])
for j in users:
userList.append(j)
userList = uniqueElements(userList)
if not len(userList) > 0:
return {"success":"false","error":"Can not find a person resposible for user administration, your request can not be approuved"}
mails = list()
mail2name = dict()
for i in userList:
tmpmail = gConfig.getValue("/Registry/Users/%s/Email" % i,"")
if tmpmail:
mails.append(tmpmail)
else:
gLogger.error("Can not find value for option /Registry/Users/%s/Email" % i)
mails = uniqueElements(mails)
if not len(mails) > 0:
return {"success":"false","error":"Can not find an email of the person resposible for the users administration, your request can not be approuved"}
gLogger.info("Admins emails: ",mails)
if not len(mails) > 0:
return {"success":"false","error":"Can not find any emails of DIRAC Administrators"}
allUsers = gConfig.getSections("/Registry/Users")
if not allUsers["OK"]:
return {"success":"false","error":"No users found at this DIRAC installation"}
allUsers = allUsers["Value"]
mail2name = dict()
for i in allUsers:
tmpmail = gConfig.getValue("/Registry/Users/%s/Email" % i,"")
if tmpmail in mails:
mail2name[tmpmail] = gConfig.getValue("/Registry/Users/%s/FullName" % i,i)
sentFailed = list()
sentSuccess = list()
errorMessage = list()
ntc = NotificationClient( getRPCClient )
for i in mails:
i = i.strip()
result = ntc.sendMail(i,"New user has registered",body,userMail,False)
if not result["OK"]:
sentFailed.append(mail2name[i])
errorMessage.append(result["Message"])
else:
sentSuccess.append(mail2name[i])
gLogger.info("Sent success: ",sentSuccess)
gLogger.info("Sent failure: ",sentFailed)
errorMessage = uniqueElements(errorMessage)
if len(sentSuccess) == 0:
if not len(errorMessage) > 0:
return {"success":"false","error":"No messages were sent to administrators due techincal reasons"}
errorMessage = ", ".join(errorMessage)
return {"success":"false","error":errorMessage}
sName = ", ".join(sentSuccess)
fName = ", ".join(sentFailed)
if len(sentFailed) > 0:
return {"success":"true","result":"Your registration request were sent successfuly to %s. Failed to sent request to %s." % (sName, fName)}
return {"success":"true","result":"Your registration request were sent successfuly to %s." % sName}
def _toJDL( self, xmlFile = '', jobDescriptionObject = None ): # messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
arguments = []
scriptName = 'jobDescription.xml'
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
if self.script:
if os.path.exists( self.script ):
scriptName = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptName )
else:
self.log.error( "File not found", self.script )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptName = xmlFile
self.addToInputSandbox.append( scriptName )
elif isinstance( jobDescriptionObject, StringIO.StringIO ):
self.log.verbose( "jobDescription is passed in as a StringIO object" )
else:
self.log.error( "Where's the job description?" )
arguments.append( os.path.basename( scriptName ) )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
if self.parametricWFArguments:
for name, value in self.parametricWFArguments.items():
arguments.append( "-p %s='%s'" % ( name, value ) )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle parameter sequences
if self.numberOfParameters > 0:
paramsDict, arguments = self._handleParameterSequences( paramsDict, arguments )
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.iteritems():
ptype = props['type']
value = props['value']
if isinstance( value, basestring) and re.search( ';', value ):
value = value.split( ';' )
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
if re.search( '^JDL', ptype ):
if isinstance( value, list ):
if isinstance( value[0], list ):
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorInt( name, value )
elif isinstance( value, basestring ) and value:
classadJob.insertAttributeInt( name, value )
elif isinstance( value, ( int, long, float ) ):
classadJob.insertAttributeInt( name, value )
if self.numberOfParameters > 0:
classadJob.insertAttributeInt( 'Parameters', self.numberOfParameters )
for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove( fToBeRemoved )
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def _toJDL(self, xmlFile='', jobDescriptionObject=None):
""" Creates a JDL representation of itself as a Job.
Example usage:
>>> job = Job()
>>> job._toJDL()
:param xmlFile: location of the XML file
:type xmlFile: str
:param jobDescriptionObject: if not None, it must be a StringIO object
:type jobDescriptionObject: StringIO
:returns: JDL (str)
"""
# Check if we have to do old bootstrap...
classadJob = ClassAd('[]')
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type': param.getType(), 'value': param.getValue()}
arguments = []
scriptName = 'jobDescription.xml'
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
# Messy but need to account for xml file being in /tmp/guid dir
if self.script:
if os.path.exists(self.script):
scriptName = os.path.abspath(self.script)
self.log.verbose('Found script name %s' % scriptName)
else:
self.log.warn("File not found", self.script)
else:
if xmlFile:
if os.path.exists(xmlFile):
self.log.verbose('Found XML File %s' % xmlFile)
scriptName = xmlFile
else:
self.log.warn("File not found", xmlFile)
else:
if os.path.exists('jobDescription.xml'):
scriptName = os.path.abspath('jobDescription.xml')
self.log.verbose('Found script name %s' % scriptName)
else:
self.log.warn("Job description XML file not found")
self.addToInputSandbox.append(scriptName)
elif isinstance(jobDescriptionObject, StringIO.StringIO):
self.log.verbose("jobDescription is passed in as a StringIO object")
else:
self.log.error("Where's the job description?")
arguments.append(os.path.basename(scriptName))
if 'LogLevel' in paramsDict:
if paramsDict['LogLevel']['value']:
arguments.append('-o LogLevel=%s' % (paramsDict['LogLevel']['value']))
else:
self.log.warn('Job LogLevel defined with null value')
if 'DIRACSetup' in paramsDict:
if paramsDict['DIRACSetup']['value']:
arguments.append('-o DIRAC/Setup=%s' % (paramsDict['DIRACSetup']['value']))
else:
self.log.warn('Job DIRACSetup defined with null value')
if 'JobMode' in paramsDict:
if paramsDict['JobMode']['value']:
arguments.append('-o JobMode=%s' % (paramsDict['JobMode']['value']))
else:
self.log.warn('Job Mode defined with null value')
if 'JobConfigArgs' in paramsDict:
if paramsDict['JobConfigArgs']['value']:
arguments.append('%s' % (paramsDict['JobConfigArgs']['value']))
else:
self.log.warn('JobConfigArgs defined with null value')
if self.parametricWFArguments:
for name, value in self.parametricWFArguments.iteritems():
arguments.append("-p %s='%s'" % (name, value))
classadJob.insertAttributeString('Executable', self.executable)
self.addToOutputSandbox.append(self.stderr)
self.addToOutputSandbox.append(self.stdout)
# Extract i/o sandbox parameters from steps and any input data parameters
# to do when introducing step-level api...
# To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join(self.addToInputSandbox)
if 'InputSandbox' in paramsDict:
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(';'))
paramsDict['InputSandbox']['value'] = ';'.join(uniqueInputSandbox)
self.log.verbose('Final unique Input Sandbox %s' % (';'.join(uniqueInputSandbox)))
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join(self.addToOutputSandbox)
if 'OutputSandbox' in paramsDict:
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(';'))
paramsDict['OutputSandbox']['value'] = ';'.join(uniqueOutputSandbox)
self.log.verbose('Final unique Output Sandbox %s' % (';'.join(uniqueOutputSandbox)))
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join(self.addToInputData)
if 'InputData' in paramsDict:
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements(finalInputData.split(';'))
paramsDict['InputData']['value'] = ';'.join(uniqueInputData)
self.log.verbose('Final unique Input Data %s' % (';'.join(uniqueInputData)))
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle parameter sequences
if self.numberOfParameters > 0:
paramsDict, arguments = self._handleParameterSequences(paramsDict, arguments)
classadJob.insertAttributeString('Arguments', ' '.join(arguments))
# Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.iteritems():
ptype = props['type']
value = props['value']
if isinstance(value, basestring) and re.search(';', value):
value = value.split(';')
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose('Found existing requirements: %s' % (value))
if re.search('^JDL', ptype):
if isinstance(value, list):
if isinstance(value[0], list):
classadJob.insertAttributeVectorStringList(name, value)
else:
classadJob.insertAttributeVectorInt(name, value)
elif isinstance(value, basestring) and value:
classadJob.insertAttributeInt(name, value)
elif isinstance(value, (int, long, float)):
classadJob.insertAttributeInt(name, value)
if self.numberOfParameters > 0:
classadJob.insertAttributeInt('Parameters', self.numberOfParameters)
for fToBeRemoved in [scriptName, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove(fToBeRemoved)
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find('[')
end = jdl.rfind(']')
return jdl[(start + 1):(end - 1)]
for fname in fileList:
if fname.endswith( '.bak' ):
gLogger.verbose( 'Ignoring BAK file: %s' % fname )
elif tarfile.is_tarfile( fname ):
try:
gLogger.debug( 'Unpacking catalog XML file %s' % ( os.path.join( directory, fname ) ) )
tarFile = tarfile.open( os.path.join( directory, fname ), 'r' )
for member in tarFile.getmembers():
tarFile.extract( member, directory )
poolCatalogList.append( os.path.join( directory, member.name ) )
except Exception, x :
gLogger.error( 'Could not untar %s with exception %s' % ( fname, str( x ) ) )
else:
poolCatalogList.append( fname )
poolCatalogList = uniqueElements( poolCatalogList )
#Now have list of all XML files but some may not be Pool XML catalogs...
finalCatList = []
for possibleCat in poolCatalogList:
try:
_cat = PoolXMLCatalog( possibleCat )
finalCatList.append( possibleCat )
except Exception, x:
gLogger.debug( 'Ignoring non-POOL catalogue file %s' % possibleCat )
gLogger.debug( 'Final list of catalog files are: %s' % ', '.join( finalCatList ) )
return finalCatList
#############################################################################
def obtainWMSJobIDs(self, transformation, fileDict, selectDelay, wmsStatusList):
""" Group files by the corresponding WMS jobIDs, check the corresponding
jobs have not been updated for the delay time. Can't get into any
mess because we start from files only in MaxReset / Assigned and check
corresponding jobs. Mixtures of files for jobs in MaxReset and Assigned
statuses only possibly include some files in Unused status (not Processed
for example) that will not be touched.
"""
prodJobIDs = uniqueElements(fileDict.values())
self.log.info('The following %s production jobIDs apply to the selected files:\n%s' % (len(prodJobIDs), prodJobIDs))
jobFileDict = {}
condDict = {'TransformationID' : transformation, self.taskIDName : prodJobIDs}
delta = datetime.timedelta( hours = selectDelay )
now = dateTime()
olderThan = now-delta
res = self.prodDB.getTransformationTasks(condDict = condDict, older = olderThan,
timeStamp = 'LastUpdateTime', inputVector = True)
self.log.debug(res)
if not res['OK']:
self.log.error('getTransformationTasks returned an error:\n%s')
return res
for jobDict in res['Value']:
missingKey = False
for key in [self.taskIDName, self.externalID, 'LastUpdateTime', self.externalStatus, 'InputVector']:
if not jobDict.has_key(key):
self.log.info('Missing key %s for job dictionary, the following is available:\n%s' % (key, jobDict))
missingKey = True
continue
if missingKey:
continue
job = jobDict[self.taskIDName]
wmsID = jobDict[self.externalID]
lastUpdate = jobDict['LastUpdateTime']
wmsStatus = jobDict[self.externalStatus]
jobInputData = jobDict['InputVector']
jobInputData = [lfn.replace('LFN:','') for lfn in jobInputData.split(';')]
if not int(wmsID):
self.log.info('Prod job %s status is %s (ID = %s) so will not recheck with WMS' %(job, wmsStatus, wmsID))
continue
self.log.info('Job %s, prod job %s last update %s, production management system status %s' % (wmsID, job, lastUpdate, wmsStatus))
#Exclude jobs not having appropriate WMS status - have to trust that production management status is correct
if not wmsStatus in wmsStatusList:
self.log.info('Job %s is in status %s, not %s so will be ignored' % (wmsID, wmsStatus, string.join(wmsStatusList, ', ')))
continue
finalJobData = []
#Must map unique files -> jobs in expected state
for lfn,prodID in fileDict.items():
if int(prodID) == int(job):
finalJobData.append(lfn)
self.log.info('Found %s files for job %s' % (len(finalJobData), job))
jobFileDict[wmsID] = finalJobData
return S_OK(jobFileDict)
def registerUser(self, paramcopy):
# Unfortunately there is no way to get rid of empty text values in JS, so i have to hardcode it on server side. Hate it!
default_values = [
"John Smith", "jsmith", "*****@*****.**",
"+33 9 10 00 10 00", "Select prefered virtual organization(s)"
]
default_values.append("Select your country")
default_values.append(
"Any additional information you want to provide to administrators")
dn = getUserDN()
username = getUsername()
if not username == "anonymous":
return {
"success":
"false",
"error":
"You are already registered in DIRAC with username: %s" %
username
}
else:
if not dn:
return {
"success":
"false",
"error":
"You have to load certificate to your browser before trying to register"
}
body = ""
userMail = False
vo = []
for i in paramcopy:
if not paramcopy[i] in default_values:
if i == "email":
userMail = paramcopy[i]
if i == "vo":
vo = paramcopy[i].split(",")
body = body + str(i) + ' - "' + str(paramcopy[i]) + '"\n'
if not userMail:
return {
"success": "false",
"error": "Can not get your email from the request"
}
gLogger.info("!!! VO: ", vo)
# TODO Check for previous requests
if not len(vo) > 0:
mails = gConfig.getValue("/Website/UserRegistrationEmail", [])
else:
mails = []
for i in vo:
i = i.strip()
voadm = gConfig.getValue("/Registry/VO/%s/VOAdmin" % i, "")
failsafe = False
if voadm:
tmpmail = gConfig.getValue(
"/Registry/Users/%s/Email" % voadm, "")
if tmpmail:
mails.append(tmpmail)
else:
gLogger.error(
"Can not find value for option /Registry/Users/%s/Email Trying failsafe option"
% voadm)
failsafe = True
else:
gLogger.error(
"Can not find value for option /Registry/VO/%s/VOAdmin Trying failsafe option"
% i)
failsafe = True
if failsafe:
failsafe = gConfig.getValue(
"/Website/UserRegistrationEmail", [])
if len(failsafe) > 0:
for j in failsafe:
mails.append(j)
else:
gLogger.error(
"Can not find value for failsafe option /Website/UserRegistrationEmail User registration for VO %s is failed"
% i)
mails = uniqueElements(mails)
if not len(mails) > 0:
groupList = list()
allGroups = gConfig.getSections("/Registry/Groups")
if not allGroups["OK"]:
return {
"success": "false",
"error": "No groups found at this DIRAC installation"
}
allGroups = allGroups["Value"]
for j in allGroups:
props = getProperties(j)
if "UserAdministrator" in props: # property which usd for user administration
groupList.append(j)
groupList = uniqueElements(groupList)
if not len(groupList) > 0:
return {
"success": "false",
"error":
"No groups, resposible for user administration, found"
}
userList = list()
for i in groupList:
users = gConfig.getValue("/Registry/Groups/%s/Users" % i, [])
for j in users:
userList.append(j)
userList = uniqueElements(userList)
if not len(userList) > 0:
return {
"success":
"false",
"error":
"Can not find a person resposible for user administration, your request can not be approuved"
}
mails = list()
mail2name = dict()
for i in userList:
tmpmail = gConfig.getValue("/Registry/Users/%s/Email" % i, "")
if tmpmail:
mails.append(tmpmail)
else:
gLogger.error(
"Can not find value for option /Registry/Users/%s/Email"
% i)
mails = uniqueElements(mails)
if not len(mails) > 0:
return {
"success":
"false",
"error":
"Can not find an email of the person resposible for the users administration, your request can not be approuved"
}
gLogger.info("Admins emails: ", mails)
if not len(mails) > 0:
return {
"success": "false",
"error": "Can not find any emails of DIRAC Administrators"
}
allUsers = gConfig.getSections("/Registry/Users")
if not allUsers["OK"]:
return {
"success": "false",
"error": "No users found at this DIRAC installation"
}
allUsers = allUsers["Value"]
mail2name = dict()
for i in allUsers:
tmpmail = gConfig.getValue("/Registry/Users/%s/Email" % i, "")
if tmpmail in mails:
mail2name[tmpmail] = gConfig.getValue(
"/Registry/Users/%s/FullName" % i, i)
sentFailed = list()
sentSuccess = list()
errorMessage = list()
ntc = NotificationClient(getRPCClient)
for i in mails:
i = i.strip()
result = ntc.sendMail(i, "New user has registered", body, userMail,
False)
if not result["OK"]:
sentFailed.append(mail2name[i])
errorMessage.append(result["Message"])
else:
sentSuccess.append(mail2name[i])
gLogger.info("Sent success: ", sentSuccess)
gLogger.info("Sent failure: ", sentFailed)
errorMessage = uniqueElements(errorMessage)
if len(sentSuccess) == 0:
if not len(errorMessage) > 0:
return {
"success":
"false",
"error":
"No messages were sent to administrators due techincal reasons"
}
errorMessage = ", ".join(errorMessage)
return {"success": "false", "error": errorMessage}
sName = ", ".join(sentSuccess)
fName = ", ".join(sentFailed)
if len(sentFailed) > 0:
return {
"success":
"true",
"result":
"Your registration request were sent successfuly to %s. Failed to sent request to %s."
% (sName, fName)
}
return {
"success":
"true",
"result":
"Your registration request were sent successfuly to %s." % sName
}
def registerUser(self):
"""
This function is used to notify DIRAC admins about user registration request
The logic is simple:
0) Check if request from this e-mail has already registered or not
1) Send mail to VO admin of requested VO
2) Send mail to users in group with UserAdministrator property
3) Send mail to users indicated in /Website/UserRegistrationAdmin option
"""
gLogger.info("Start processing a registration request")
checkUserCredentials()
# Check for having a DN but no username
dn = getUserDN()
if not dn:
error = "Certificate is not loaded in the browser or DN is absent"
gLogger.error("Service response: %s" % error)
return {"success": "false", "error": error}
username = getUsername()
if not username == "anonymous":
error = "You are already registered in DIRAC with username: %s" % username
gLogger.error("Service response: %s" % error)
return {"success": "false", "error": error}
gLogger.info("DN: %s" % dn)
if not "email" in request.params:
error = "Can not get your email address from the request"
gLogger.debug("Service response: %s" % error)
return {"success": "false", "error": error}
userMail = request.params["email"]
result = self.isRequested(userMail)
gLogger.debug(result)
if result["OK"]:
return render("/reg_done.mako")
result = self.registerRequest(dn, userMail)
gLogger.debug(result)
if not result["OK"]:
return {"success": "false", "error": result["Message"]}
vo = fromChar(request.params["vo"])
if not vo:
error = "You should indicate a VirtualOrganization for membership"
gLogger.debug("Service response: %s" % error)
return {"success": "false", "error": error}
gLogger.info("User want to be register in VO(s): %s" % vo)
body = str()
for i in request.params:
if not i in ["registration_request", "email", "vo"]:
text = self.checkUnicode(request.params[i])
info = "%s - %s" % (i, text)
body = body + info + "\n"
body = body + "DN - " + dn
gLogger.debug("email body: %s" % body)
adminList = self.__getAdminList(vo)
if not len(adminList) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug("Service response: %s" % error)
return {"success": "false", "error": error}
adminList = uniqueElements(adminList)
gLogger.info("Chosen admin(s): %s" % adminList)
sendDict = self.getMailDict(adminList)
if not len(sendDict) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug("Service response: %s" % error)
return {"success": "false", "error": error}
gLogger.debug("Final dictionary with mails to be used %s" % sendDict)
if socket.gethostname().find('.') >= 0:
hostname = socket.gethostname()
else:
hostname = socket.gethostbyaddr(socket.gethostname())[0]
title = "New user has sent registration request to %s" % hostname
return self.sendMail(sendDict, title, body, userMail)
def _toJDL(
self, xmlFile="", jobDescriptionObject=None
): # messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
# Check if we have to do old bootstrap...
classadJob = ClassAd("[]")
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {"type": param.getType(), "value": param.getValue()}
arguments = []
scriptname = "jobDescription.xml"
if jobDescriptionObject is None:
# if we are here it's because there's a real file, on disk, that is named 'jobDescription.xml'
if self.script:
if os.path.exists(self.script):
scriptname = os.path.abspath(self.script)
self.log.verbose("Found script name %s" % scriptname)
else:
self.log.error("File not found", self.script)
else:
if xmlFile:
self.log.verbose("Found XML File %s" % xmlFile)
scriptname = xmlFile
self.addToInputSandbox.append(scriptname)
elif isinstance(jobDescriptionObject, StringIO.StringIO):
self.log.verbose("jobDescription is passed in as a StringIO object")
else:
self.log.error("Where's the job description?")
arguments.append(os.path.basename(scriptname))
if paramsDict.has_key("LogLevel"):
if paramsDict["LogLevel"]["value"]:
arguments.append("-o LogLevel=%s" % (paramsDict["LogLevel"]["value"]))
else:
self.log.warn("Job LogLevel defined with null value")
if paramsDict.has_key("DIRACSetup"):
if paramsDict["DIRACSetup"]["value"]:
arguments.append("-o DIRAC/Setup=%s" % (paramsDict["DIRACSetup"]["value"]))
else:
self.log.warn("Job DIRACSetup defined with null value")
if paramsDict.has_key("JobMode"):
if paramsDict["JobMode"]["value"]:
arguments.append("-o JobMode=%s" % (paramsDict["JobMode"]["value"]))
else:
self.log.warn("Job Mode defined with null value")
if paramsDict.has_key("JobConfigArgs"):
if paramsDict["JobConfigArgs"]["value"]:
arguments.append("%s" % (paramsDict["JobConfigArgs"]["value"]))
else:
self.log.warn("JobConfigArgs defined with null value")
classadJob.insertAttributeString("Executable", self.executable)
self.addToOutputSandbox.append(self.stderr)
self.addToOutputSandbox.append(self.stdout)
# Extract i/o sandbox parameters from steps and any input data parameters
# to do when introducing step-level api...
# To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ";".join(self.addToInputSandbox)
if paramsDict.has_key("InputSandbox"):
currentFiles = paramsDict["InputSandbox"]["value"]
finalInputSandbox = currentFiles + ";" + extraFiles
uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";"))
paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox)
self.log.verbose("Final unique Input Sandbox %s" % (";".join(uniqueInputSandbox)))
else:
paramsDict["InputSandbox"] = {}
paramsDict["InputSandbox"]["value"] = extraFiles
paramsDict["InputSandbox"]["type"] = "JDL"
if self.addToOutputSandbox:
extraFiles = ";".join(self.addToOutputSandbox)
if paramsDict.has_key("OutputSandbox"):
currentFiles = paramsDict["OutputSandbox"]["value"]
finalOutputSandbox = currentFiles + ";" + extraFiles
uniqueOutputSandbox = uniqueElements(finalOutputSandbox.split(";"))
paramsDict["OutputSandbox"]["value"] = ";".join(uniqueOutputSandbox)
self.log.verbose("Final unique Output Sandbox %s" % (";".join(uniqueOutputSandbox)))
else:
paramsDict["OutputSandbox"] = {}
paramsDict["OutputSandbox"]["value"] = extraFiles
paramsDict["OutputSandbox"]["type"] = "JDL"
if self.addToInputData:
extraFiles = ";".join(self.addToInputData)
if paramsDict.has_key("InputData"):
currentFiles = paramsDict["InputData"]["value"]
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ";" + extraFiles
uniqueInputData = uniqueElements(finalInputData.split(";"))
paramsDict["InputData"]["value"] = ";".join(uniqueInputData)
self.log.verbose("Final unique Input Data %s" % (";".join(uniqueInputData)))
else:
paramsDict["InputData"] = {}
paramsDict["InputData"]["value"] = extraFiles
paramsDict["InputData"]["type"] = "JDL"
# Handle here the Parametric values
if self.parametric:
for pType in ["InputData", "InputSandbox"]:
if self.parametric.has_key(pType):
if paramsDict.has_key(pType) and paramsDict[pType]["value"]:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]["value"].split(";")
tmpList = []
if type(pData[0]) == list:
for pElement in pData:
tmpList.append(currentFiles + pElement)
else:
for pElement in pData:
tmpList.append(currentFiles + [pElement])
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]["value"] = "%s"
paramsDict[pType]["type"] = "JDL"
self.parametric["files"] = self.parametric[pType]
arguments.append(" -p Parametric" + pType + "=%s")
break
if self.parametric.has_key("files"):
paramsDict["Parameters"] = {}
paramsDict["Parameters"]["value"] = self.parametric["files"]
paramsDict["Parameters"]["type"] = "JDL"
if self.parametric.has_key("GenericParameters"):
paramsDict["Parameters"] = {}
paramsDict["Parameters"]["value"] = self.parametric["GenericParameters"]
paramsDict["Parameters"]["type"] = "JDL"
arguments.append(" -p ParametricParameters=%s")
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString("Arguments", " ".join(arguments))
# Add any JDL parameters to classad obeying lists with ';' rule
for name, props in paramsDict.items():
ptype = props["type"]
value = props["value"]
if name.lower() == "requirements" and ptype == "JDL":
self.log.verbose("Found existing requirements: %s" % (value))
if re.search("^JDL", ptype):
if type(value) == list:
if type(value[0]) == list:
classadJob.insertAttributeVectorStringList(name, value)
else:
classadJob.insertAttributeVectorString(name, value)
elif value == "%s":
classadJob.insertAttributeInt(name, value)
elif not re.search(";", value) or name == "GridRequirements": # not a nice fix...
classadJob.insertAttributeString(name, value)
else:
classadJob.insertAttributeVectorString(name, value.split(";"))
for fToBeRemoved in [scriptname, self.stdout, self.stderr]:
try:
self.addToInputSandbox.remove(fToBeRemoved)
except ValueError:
pass
jdl = classadJob.asJDL()
start = jdl.find("[")
end = jdl.rfind("]")
return jdl[(start + 1) : (end - 1)]
def __createQuery(self,args):
""" Create the metadata query out of the command line arguments
"""
argss = args.split()
result = self.fc.getMetadataFields()
if not result['OK']:
print ("Error: %s" % result['Message'])
return None
if not result['Value']:
print "Error: no metadata fields defined"
return None
typeDict = result['Value']
metaDict = {}
contMode = False
for arg in argss:
if not contMode:
operation = ''
for op in ['>','<','>=','<=','!=','=']:
if arg.find(op) != -1:
operation = op
break
if not operation:
print "Error: operation is not found in the query"
return None
name,value = arg.split(operation)
if not name in typeDict:
print "Error: metadata field %s not defined" % name
return None
mtype = typeDict[name]
else:
value += ' ' + arg
value = value.replace(contMode,'')
contMode = False
if value[0] == '"' or value[0] == "'":
if value[-1] != '"' and value != "'":
contMode = value[0]
continue
if value.find(',') != -1:
valueList = [ x.replace("'","").replace('"','') for x in value.split(',') ]
mvalue = valueList
if mtype[0:3].lower() == 'int':
mvalue = [ int(x) for x in valueList if not x in ['Missing','Any'] ]
mvalue += [ x for x in valueList if x in ['Missing','Any'] ]
if mtype[0:5].lower() == 'float':
mvalue = [ float(x) for x in valueList if not x in ['Missing','Any'] ]
mvalue += [ x for x in valueList if x in ['Missing','Any'] ]
if operation == "=":
operation = 'in'
if operation == "!=":
operation = 'nin'
mvalue = {operation:mvalue}
else:
mvalue = value.replace("'","").replace('"','')
if not value in ['Missing','Any']:
if mtype[0:3].lower() == 'int':
mvalue = int(value)
if mtype[0:5].lower() == 'float':
mvalue = float(value)
if operation != '=':
mvalue = {operation:mvalue}
if name in metaDict:
if type(metaDict[name]) == DictType:
if type(mvalue) == DictType:
op,value = mvalue.items()[0]
if op in metaDict[name]:
if type(metaDict[name][op]) == ListType:
if type(value) == ListType:
metaDict[name][op] = uniqueElements(metaDict[name][op] + value)
else:
metaDict[name][op] = uniqueElements(metaDict[name][op].append(value))
else:
if type(value) == ListType:
metaDict[name][op] = uniqueElements([metaDict[name][op]] + value)
else:
metaDict[name][op] = uniqueElements([metaDict[name][op],value])
else:
metaDict[name].update(mvalue)
else:
if type(mvalue) == ListType:
metaDict[name].update({'in':mvalue})
else:
metaDict[name].update({'=':mvalue})
elif type(metaDict[name]) == ListType:
if type(mvalue) == DictType:
metaDict[name] = {'in':metaDict[name]}
metaDict[name].update(mvalue)
elif type(mvalue) == ListType:
metaDict[name] = uniqueElements(metaDict[name] + mvalue)
else:
metaDict[name] = uniqueElements(metaDict[name].append(mvalue))
else:
if type(mvalue) == DictType:
metaDict[name] = {'=':metaDict[name]}
metaDict[name].update(mvalue)
elif type(mvalue) == ListType:
metaDict[name] = uniqueElements([metaDict[name]] + mvalue)
else:
metaDict[name] = uniqueElements([metaDict[name],mvalue])
else:
metaDict[name] = mvalue
return metaDict
def __getSiteCandidates(self, inputData):
"""This method returns a list of possible site candidates based on the
job input data requirement. For each site candidate, the number of files
on disk and tape is resolved.
"""
fileSEs = {}
for lfn, replicas in inputData.items():
siteList = []
for se in replicas.keys():
sites = self.__getSitesForSE(se)
if sites['OK']:
siteList += sites['Value']
fileSEs[lfn] = uniqueElements(siteList)
siteCandidates = []
i = 0
for fileName, sites in fileSEs.items():
if not i:
siteCandidates = sites
else:
tempSite = []
for site in siteCandidates:
if site in sites:
tempSite.append(site)
siteCandidates = tempSite
i += 1
if not len(siteCandidates):
return S_ERROR('No candidate sites available')
#In addition, check number of files on tape and disk for each site
#for optimizations during scheduling
siteResult = {}
for site in siteCandidates:
siteResult[site] = {'disk': [], 'tape': []}
seDict = {}
for lfn, replicas in inputData.items():
for se in replicas.keys():
if se not in seDict:
sites = self.__getSitesForSE(se)
if not sites['OK']:
continue
try:
#storageElement = StorageElement( se )
result = self.resourceStatus.getStorageElementStatus(
se, statusType='ReadAccess')
if not result['OK']:
continue
seDict[se] = {
'Sites': sites['Value'],
'SEParams': result['Value'][se]
}
result = self.resourcesHelper.getStorageElementOptionsDict(
se)
if not result['OK']:
continue
seDict[se]['SEParams'].update(result['Value'])
except Exception:
self.log.exception(
'Failed to instantiate StorageElement( %s )' % se)
continue
for site in seDict[se]['Sites']:
if site in siteCandidates:
if seDict[se]['SEParams']['ReadAccess'] and seDict[se][
'SEParams']['DiskSE']:
if lfn not in siteResult[site]['disk']:
siteResult[site]['disk'].append(lfn)
if lfn in siteResult[site]['tape']:
siteResult[site]['tape'].remove(lfn)
if seDict[se]['SEParams']['ReadAccess'] and seDict[se][
'SEParams']['TapeSE']:
if lfn not in siteResult[site][
'tape'] and lfn not in siteResult[site][
'disk']:
siteResult[site]['tape'].append(lfn)
for site in siteResult:
siteResult[site]['disk'] = len(siteResult[site]['disk'])
siteResult[site]['tape'] = len(siteResult[site]['tape'])
return S_OK(siteResult)
def _prepareRemoteHost( self, host = None ):
""" Prepare remote directories and upload control script
"""
ssh = SSH( host = host, parameters = self.ceParameters )
# Make remote directories
dirTuple = tuple ( uniqueElements( [self.sharedArea,
self.executableArea,
self.infoArea,
self.batchOutput,
self.batchError,
self.workArea] ) )
nDirs = len( dirTuple )
cmd = 'mkdir -p %s; '*nDirs % dirTuple
cmd = "bash -c '%s'" % cmd
self.log.verbose( 'Creating working directories on %s' % self.ceParameters['SSHHost'] )
result = ssh.sshCall( 30, cmd )
if not result['OK']:
self.log.warn( 'Failed creating working directories: %s' % result['Message'][1] )
return result
status, output, _error = result['Value']
if status == -1:
self.log.warn( 'Timeout while creating directories' )
return S_ERROR( 'Timeout while creating directories' )
if "cannot" in output:
self.log.warn( 'Failed to create directories: %s' % output )
return S_ERROR( 'Failed to create directories: %s' % output )
# Upload the control script now
batchSystemDir = os.path.join( rootPath, "DIRAC", "Resources", "Computing", "BatchSystems" )
batchSystemScript = os.path.join( batchSystemDir, '%s.py' % self.batchSystem )
batchSystemExecutor = os.path.join( batchSystemDir, 'executeBatch.py' )
self.log.verbose( 'Uploading %s script to %s' % ( self.batchSystem, self.ceParameters['SSHHost'] ) )
remoteScript = '%s/execute_batch' % self.sharedArea
result = ssh.scpCall( 30,
'%s %s' % ( batchSystemScript, batchSystemExecutor ),
remoteScript,
postUploadCommand = 'chmod +x %s' % remoteScript )
if not result['OK']:
self.log.warn( 'Failed uploading control script: %s' % result['Message'][1] )
return result
status, output, _error = result['Value']
if status != 0:
if status == -1:
self.log.warn( 'Timeout while uploading control script' )
return S_ERROR( 'Timeout while uploading control script' )
else:
self.log.warn( 'Failed uploading control script: %s' % output )
return S_ERROR( 'Failed uploading control script' )
# Chmod the control scripts
#self.log.verbose( 'Chmod +x control script' )
#result = ssh.sshCall( 10, "chmod +x %s/%s" % ( self.sharedArea, self.controlScript ) )
#if not result['OK']:
# self.log.warn( 'Failed chmod control script: %s' % result['Message'][1] )
# return result
#status, output, _error = result['Value']
#if status != 0:
# if status == -1:
# self.log.warn( 'Timeout while chmod control script' )
# return S_ERROR( 'Timeout while chmod control script' )
# else:
# self.log.warn( 'Failed uploading chmod script: %s' % output )
# return S_ERROR( 'Failed uploading chmod script' )
return S_OK()
def __getSiteCandidates( self, inputData ):
"""This method returns a list of possible site candidates based on the
job input data requirement. For each site candidate, the number of files
on disk and tape is resolved.
"""
fileSEs = {}
for lfn, replicas in inputData.items():
siteList = []
for se in replicas.keys():
sites = self.__getSitesForSE( se )
if sites['OK']:
siteList += sites['Value']
fileSEs[lfn] = uniqueElements( siteList )
siteCandidates = []
i = 0
for _fileName, sites in fileSEs.items():
if not i:
siteCandidates = sites
else:
tempSite = []
for site in siteCandidates:
if site in sites:
tempSite.append( site )
siteCandidates = tempSite
i += 1
if not len( siteCandidates ):
return S_ERROR( 'No candidate sites available' )
#In addition, check number of files on tape and disk for each site
#for optimizations during scheduling
siteResult = {}
for site in siteCandidates:
siteResult[site] = { 'disk': [], 'tape': [] }
seDict = {}
for lfn, replicas in inputData.items():
for se in replicas.keys():
if se not in seDict:
sites = self.__getSitesForSE( se )
if not sites['OK']:
continue
try:
#storageElement = StorageElement( se )
result = self.resourceStatus.getStorageElementStatus( se, statusType = 'ReadAccess' )
if not result['OK']:
continue
seDict[se] = { 'Sites': sites['Value'], 'SEParams': result['Value'][se] }
result = getStorageElementOptions( se )
if not result['OK']:
continue
seDict[se]['SEParams'].update(result['Value'])
except Exception:
self.log.exception( 'Failed to instantiate StorageElement( %s )' % se )
continue
for site in seDict[se]['Sites']:
if site in siteCandidates:
if seDict[se]['SEParams']['ReadAccess'] and seDict[se]['SEParams']['DiskSE']:
if lfn not in siteResult[site]['disk']:
siteResult[site]['disk'].append( lfn )
if lfn in siteResult[site]['tape']:
siteResult[site]['tape'].remove( lfn )
if seDict[se]['SEParams']['ReadAccess'] and seDict[se]['SEParams']['TapeSE']:
if lfn not in siteResult[site]['tape'] and lfn not in siteResult[site]['disk']:
siteResult[site]['tape'].append( lfn )
for site in siteResult:
siteResult[site]['disk'] = len( siteResult[site]['disk'] )
siteResult[site]['tape'] = len( siteResult[site]['tape'] )
return S_OK( siteResult )
def registerUser(self,paramcopy):
gLogger.info("Start processing a registration request")
"""
Unfortunately there is no way to get rid of empty text values in JS,
so i have to hardcode it on server side. Hate it!
"""
default_values = ["John Smith","jsmith","*****@*****.**","+33 9 10 00 10 00"]
default_values.append("Select preferred virtual organization(s)")
default_values.append("Select your country")
default_values.append("Any additional information you want to provide to administrators")
# Check for having a DN but no username
dn = getUserDN()
username = getUsername()
gLogger.debug("User's DN: %s and DIRAC username: %s" % (dn, username))
if not username == "anonymous":
error = "You are already registered in DIRAC with username: %s" % username
gLogger.debug("Service response: %s" % error)
return {"success":"false","error":error}
else:
if not dn:
error = "Certificate is not loaded to a browser or DN is absent"
gLogger.debug("Service response: %s" % error)
return {"success":"false","error":error}
body = ""
userMail = False
vo = []
# Check for user's email, creating mail body
gLogger.debug("Request's body:")
for i in paramcopy:
gLogger.debug("%s - %s" % (i,paramcopy[i]))
if not paramcopy[i] in default_values:
if i == "email":
userMail = paramcopy[i]
if i == "vo":
vo = paramcopy[i].split(",")
body = body + str(i) + ' - "' + str(paramcopy[i]) + '"\n'
if not userMail:
error = "Can not get your email address from the request"
gLogger.debug("Service response: %s" % error)
return {"success":"false","error":error}
gLogger.info("User want to be register in VO(s): %s" % vo)
# TODO Check for previous requests
# Get admin mail based on requested VO i.e. mail of VO admin
mails = list()
gLogger.debug("Trying to get admin username to take care about request")
for i in vo:
gLogger.debug("VOAdmin for VO: %s" % i)
i = i.strip()
voadm = gConfig.getValue("/Registry/VO/%s/VOAdmin" % i,[])
gLogger.debug("/Registry/VO/%s/VOAdmin - %s" % (i,voadm))
for user in voadm:
mails.append(user)
# If no VOAdmin - try to get admin mails based on group properties
if not len(mails) > 0:
gLogger.debug("No VO admins found. Trying to get something based on group property")
groupList = list()
groups = gConfig.getSections("/Registry/Groups")
gLogger.debug("Group response: %s" % groups)
if groups["OK"]:
allGroups = groups["Value"]
gLogger.debug("Looking for UserAdministrator property")
for j in allGroups:
props = getProperties(j)
gLogger.debug("%s properties: %s" % (j,props)) #1
if "UserAdministrator" in props: # property which is used for user administration
groupList.append(j)
groupList = uniqueElements(groupList)
gLogger.debug("Chosen group(s): %s" % groupList)
if len(groupList) > 0:
for i in groupList:
users = gConfig.getValue("/Registry/Groups/%s/Users" % i,[])
gLogger.debug("%s users: %s" % (i,users))
for user in users:
mails.append(user)
# Last stand - Failsafe option
if not len(mails) > 0:
gLogger.debug("No suitable groups found. Trying failsafe")
regAdmin = gConfig.getValue("/Website/UserRegistrationAdmin",[])
gLogger.debug("/Website/UserRegistrationAdmin - %s" % regAdmin)
for user in regAdmin:
mails.append(user)
mails = uniqueElements(mails)
gLogger.info("Chosen admin(s): %s" % mails)
# Final check of usernames
if not len(mails) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug("Service response: %s" % error)
return {"success":"false","error":error}
# Convert usernames to { e-mail : full name }
gLogger.debug("Trying to get admin's mail and associated name")
sendDict = dict()
for user in mails:
email = gConfig.getValue("/Registry/Users/%s/Email" % user,"")
gLogger.debug("/Registry/Users/%s/Email - '%s'" % (user,email))
emil = email.strip()
if not email:
gLogger.error("Can't find value for option /Registry/Users/%s/Email" % user)
continue
fname = gConfig.getValue("/Registry/Users/%s/FullName" % user,"")
gLogger.debug("/Registry/Users/%s/FullName - '%s'" % (user,fname))
fname = fname.strip()
if not fname:
fname = user
gLogger.debug("FullName is absent, name to be used: %s" % fname)
sendDict[email] = fname
# Final check of mails
gLogger.debug("Final dictionary with mails to be used %s" % sendDict)
if not len(sendDict) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug("Service response: %s" % error)
return {"success":"false","error":error}
# Sending a mail
sentSuccess = list()
sentFailed = list()
gLogger.debug("Initializing Notification client")
ntc = NotificationClient(lambda x, timeout: getRPCClient(x, timeout=timeout, static = True) )
gLogger.debug("Sending messages")
for email,name in sendDict.iteritems():
gLogger.debug("ntc.sendMail(%s,New user has registered,%s,%s,False" % (email,body,userMail))
result = ntc.sendMail(email,"New user has registered",body,userMail,False)
if not result["OK"]:
error = name + ": " + result["Message"]
sentFailed.append(error)
gLogger.error("Sent failure: ", error)
else:
gLogger.info("Successfully sent to %s" % name)
sentSuccess.append(name)
# Returning results
sName = ", ".join(sentSuccess)
gLogger.info("End of processing of a registration request")
gLogger.debug("Service response sent to a user:"******"Your registration request were sent successfully to: "
result = result + sName + "\n\nFailed to sent it to:\n"
result = result + "\n".join(sentFailed)
gLogger.debug(result)
return {"success":"true","result":result}
elif len(sentSuccess) > 0 and (not len(sentFailed)) > 0:
result = "Your registration request were sent successfully to: %s" % sName
gLogger.debug(result)
return {"success":"true","result":result}
elif (not len(sentSuccess)) > 0 and len(sentFailed) > 0:
result = "Failed to sent your request to:\n"
result = result + "\n".join(sentFailed)
gLogger.debug(result)
return {"success":"false","error":result}
else:
result = "No messages were sent to administrator due technical failure"
gLogger.debug(result)
return {"success":"false","error":result}
def registerUser( self ):
"""
This function is used to notify DIRAC admins about user registration request
The logic is simple:
0) Check if request from this e-mail has already registered or not
1) Send mail to VO admin of requested VO
2) Send mail to users in group with UserAdministrator property
3) Send mail to users indicated in /Website/UserRegistrationAdmin option
"""
gLogger.info("Start processing a registration request")
checkUserCredentials()
# Check for having a DN but no username
dn = getUserDN()
if not dn:
error = "Certificate is not loaded in the browser or DN is absent"
gLogger.error( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
username = getUsername()
if not username == "anonymous":
error = "You are already registered in DIRAC with username: %s" % username
gLogger.error( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.info( "DN: %s" % dn )
if not request.params.has_key( "email" ):
error = "Can not get your email address from the request"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
userMail = request.params[ "email" ]
if self.alreadyRequested( userMail ):
error = "Request associated with %s already registered" % userMail
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
vo = fromChar( request.params[ "vo" ] )
if not vo:
error = "You should indicate a VirtualOrganization for membership"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.info( "User want to be register in VO(s): %s" % vo )
body = str()
for i in request.params:
if not i in [ "registration_request" , "email" , "vo" ]:
info = self.__checkUnicode( i , request.params[ i ] )
body = body + info + "\n"
body = body + "DN - " + dn
gLogger.debug( "email body: %s" % body )
adminList = self.__getAdminList( vo )
if not len( adminList ) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
adminList = uniqueElements( adminList )
gLogger.info( "Chosen admin(s): %s" % adminList )
sendDict = self.__getMailDict( adminList )
if not len(sendDict) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.debug( "Final dictionary with mails to be used %s" % sendDict )
return self.__sendAMail( sendDict , body , userMail )
def registerUser( self ):
"""
This function is used to notify DIRAC admins about user registration request
The logic is simple:
0) Check if request from this e-mail has already registered or not
1) Send mail to VO admin of requested VO
2) Send mail to users in group with UserAdministrator property
3) Send mail to users indicated in /Website/UserRegistrationAdmin option
"""
gLogger.info("Start processing a registration request")
checkUserCredentials()
# Check for having a DN but no username
dn = getUserDN()
if not dn:
error = "Certificate is not loaded in the browser or DN is absent"
gLogger.error( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
username = getUsername()
if not username == "anonymous":
error = "You are already registered in DIRAC with username: %s" % username
gLogger.error( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.info( "DN: %s" % dn )
if not "email" in request.params:
error = "Can not get your email address from the request"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
userMail = request.params[ "email" ]
result = self.isRequested( userMail )
gLogger.debug( result )
if result[ "OK" ]:
return render( "/reg_done.mako" )
result = self.registerRequest( dn , userMail )
gLogger.debug( result )
if not result[ "OK" ]:
return { "success" : "false" , "error" : result[ "Message" ] }
vo = fromChar( request.params[ "vo" ] )
if not vo:
error = "You should indicate a VirtualOrganization for membership"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.info( "User want to be register in VO(s): %s" % vo )
body = str()
for i in request.params:
if not i in [ "registration_request" , "email" , "vo" ]:
text = self.checkUnicode( request.params[ i ] )
info = "%s - %s" % ( i , text )
body = body + info + "\n"
body = body + "DN - " + dn
gLogger.debug( "email body: %s" % body )
adminList = self.__getAdminList( vo )
if not len( adminList ) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
adminList = uniqueElements( adminList )
gLogger.info( "Chosen admin(s): %s" % adminList )
sendDict = self.getMailDict( adminList )
if not len( sendDict ) > 0:
error = "Can't get in contact with administrators about your request\n"
error = error + "Most likely this DIRAC instance is not configured yet"
gLogger.debug( "Service response: %s" % error )
return { "success" : "false" , "error" : error }
gLogger.debug( "Final dictionary with mails to be used %s" % sendDict )
if socket.gethostname().find( '.' ) >= 0:
hostname = socket.gethostname()
else:
hostname = socket.gethostbyaddr( socket.gethostname() )[ 0 ]
title = "New user has sent registration request to %s" % hostname
return self.sendMail( sendDict , title , body , userMail )
def getDestinationSEList(outputSE, site, outputmode='Any'):
""" Evaluate the output SE list from a workflow and return the concrete list
of SEs to upload output data.
"""
# Add output SE defined in the job description
gLogger.info('Resolving workflow output SE description: %s' % outputSE)
# Check if the SE is defined explicitly for the site
prefix = site.split('.')[0]
country = site.split('.')[-1]
# Concrete SE name
result = gConfig.getOptions('/Resources/StorageElements/' + outputSE)
if result['OK']:
gLogger.info('Found concrete SE %s' % outputSE)
return S_OK([outputSE])
# There is an alias defined for this Site
alias_se = gConfig.getValue(
'/Resources/Sites/%s/%s/AssociatedSEs/%s' % (prefix, site, outputSE),
[])
if alias_se:
gLogger.info('Found associated SE for site %s' % (alias_se))
return S_OK(alias_se)
localSEs = getSEsForSite(site)['Value']
gLogger.verbose('Local SE list is: %s' % (localSEs))
groupSEs = gConfig.getValue('/Resources/StorageElementGroups/' + outputSE,
[])
gLogger.verbose('Group SE list is: %s' % (groupSEs))
if not groupSEs:
return S_ERROR('Failed to resolve SE ' + outputSE)
if outputmode.lower() == "local":
for se in localSEs:
if se in groupSEs:
gLogger.info('Found eligible local SE: %s' % (se))
return S_OK([se])
#check if country is already one with associated SEs
associatedSE = gConfig.getValue(
'/Resources/Countries/%s/AssociatedSEs/%s' % (country, outputSE),
'')
if associatedSE:
gLogger.info(
'Found associated SE %s in /Resources/Countries/%s/AssociatedSEs/%s'
% (associatedSE, country, outputSE))
return S_OK([associatedSE])
# Final check for country associated SE
count = 0
assignedCountry = country
while count < 10:
gLogger.verbose('Loop count = %s' % (count))
gLogger.verbose("/Resources/Countries/%s/AssignedTo" %
assignedCountry)
opt = gConfig.getOption("/Resources/Countries/%s/AssignedTo" %
assignedCountry)
if opt['OK'] and opt['Value']:
assignedCountry = opt['Value']
gLogger.verbose('/Resources/Countries/%s/AssociatedSEs' %
assignedCountry)
assocCheck = gConfig.getOption(
'/Resources/Countries/%s/AssociatedSEs' % assignedCountry)
if assocCheck['OK'] and assocCheck['Value']:
break
count += 1
if not assignedCountry:
return S_ERROR('Could not determine associated SE list for %s' %
country)
alias_se = gConfig.getValue(
'/Resources/Countries/%s/AssociatedSEs/%s' %
(assignedCountry, outputSE), [])
if alias_se:
gLogger.info('Found alias SE for site: %s' % alias_se)
return S_OK(alias_se)
else:
gLogger.error(
'Could not establish alias SE for country %s from section: /Resources/Countries/%s/AssociatedSEs/%s'
% (country, assignedCountry, outputSE))
return S_ERROR('Failed to resolve SE ' + outputSE)
# For collective Any and All modes return the whole group
# Make sure that local SEs are passing first
newSEList = []
for se in groupSEs:
if se in localSEs:
newSEList.append(se)
uniqueSEs = uniqueElements(newSEList + groupSEs)
gLogger.verbose('Found unique SEs: %s' % (uniqueSEs))
return S_OK(uniqueSEs)
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being in /tmp/guid dir
"""Creates a JDL representation of itself as a Job.
"""
#Check if we have to do old bootstrap...
classadJob = ClassAd( '[]' )
paramsDict = {}
params = self.workflow.parameters # ParameterCollection object
paramList = params
for param in paramList:
paramsDict[param.getName()] = {'type':param.getType(), 'value':param.getValue()}
scriptname = 'jobDescription.xml'
arguments = []
if self.script:
if os.path.exists( self.script ):
scriptname = os.path.abspath( self.script )
self.log.verbose( 'Found script name %s' % scriptname )
else:
if xmlFile:
self.log.verbose( 'Found XML File %s' % xmlFile )
scriptname = xmlFile
arguments.append( os.path.basename( scriptname ) )
self.addToInputSandbox.append( scriptname )
if paramsDict.has_key( 'LogLevel' ):
if paramsDict['LogLevel']['value']:
arguments.append( '-o LogLevel=%s' % ( paramsDict['LogLevel']['value'] ) )
else:
self.log.warn( 'Job LogLevel defined with null value' )
if paramsDict.has_key( 'DIRACSetup' ):
if paramsDict['DIRACSetup']['value']:
arguments.append( '-o DIRAC/Setup=%s' % ( paramsDict['DIRACSetup']['value'] ) )
else:
self.log.warn( 'Job DIRACSetup defined with null value' )
if paramsDict.has_key( 'JobMode' ):
if paramsDict['JobMode']['value']:
arguments.append( '-o JobMode=%s' % ( paramsDict['JobMode']['value'] ) )
else:
self.log.warn( 'Job Mode defined with null value' )
if paramsDict.has_key( 'JobConfigArgs' ):
if paramsDict['JobConfigArgs']['value']:
arguments.append( '%s' % ( paramsDict['JobConfigArgs']['value'] ) )
else:
self.log.warn( 'JobConfigArgs defined with null value' )
classadJob.insertAttributeString( 'Executable', self.executable )
self.addToOutputSandbox.append( self.stderr )
self.addToOutputSandbox.append( self.stdout )
#Extract i/o sandbox parameters from steps and any input data parameters
#to do when introducing step-level api...
#To add any additional files to input and output sandboxes
if self.addToInputSandbox:
extraFiles = ';'.join( self.addToInputSandbox )
if paramsDict.has_key( 'InputSandbox' ):
currentFiles = paramsDict['InputSandbox']['value']
finalInputSandbox = currentFiles + ';' + extraFiles
uniqueInputSandbox = uniqueElements( finalInputSandbox.split( ';' ) )
paramsDict['InputSandbox']['value'] = ';'.join( uniqueInputSandbox )
self.log.verbose( 'Final unique Input Sandbox %s' % ( ';'.join( uniqueInputSandbox ) ) )
else:
paramsDict['InputSandbox'] = {}
paramsDict['InputSandbox']['value'] = extraFiles
paramsDict['InputSandbox']['type'] = 'JDL'
if self.addToOutputSandbox:
extraFiles = ';'.join( self.addToOutputSandbox )
if paramsDict.has_key( 'OutputSandbox' ):
currentFiles = paramsDict['OutputSandbox']['value']
finalOutputSandbox = currentFiles + ';' + extraFiles
uniqueOutputSandbox = uniqueElements( finalOutputSandbox.split( ';' ) )
paramsDict['OutputSandbox']['value'] = ';'.join( uniqueOutputSandbox )
self.log.verbose( 'Final unique Output Sandbox %s' % ( ';'.join( uniqueOutputSandbox ) ) )
else:
paramsDict['OutputSandbox'] = {}
paramsDict['OutputSandbox']['value'] = extraFiles
paramsDict['OutputSandbox']['type'] = 'JDL'
if self.addToInputData:
extraFiles = ';'.join( self.addToInputData )
if paramsDict.has_key( 'InputData' ):
currentFiles = paramsDict['InputData']['value']
finalInputData = extraFiles
if currentFiles:
finalInputData = currentFiles + ';' + extraFiles
uniqueInputData = uniqueElements( finalInputData.split( ';' ) )
paramsDict['InputData']['value'] = ';'.join( uniqueInputData )
self.log.verbose( 'Final unique Input Data %s' % ( ';'.join( uniqueInputData ) ) )
else:
paramsDict['InputData'] = {}
paramsDict['InputData']['value'] = extraFiles
paramsDict['InputData']['type'] = 'JDL'
# Handle here the Parametric values
if self.parametric:
for pType in ['InputData', 'InputSandbox']:
if self.parametric.has_key( pType ):
if paramsDict.has_key( pType ) and paramsDict[pType]['value']:
pData = self.parametric[pType]
# List of lists case
currentFiles = paramsDict[pType]['value'].split( ';' )
tmpList = []
if type( pData[0] ) == list:
for pElement in pData:
tmpList.append( currentFiles + pElement )
else:
for pElement in pData:
tmpList.append( currentFiles + [pElement] )
self.parametric[pType] = tmpList
paramsDict[pType] = {}
paramsDict[pType]['value'] = "%s"
paramsDict[pType]['type'] = 'JDL'
self.parametric['files'] = self.parametric[pType]
arguments.append( ' -p Parametric' + pType + '=%s' )
break
if self.parametric.has_key( 'files' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['files']
paramsDict['Parameters']['type'] = 'JDL'
if self.parametric.has_key( 'GenericParameters' ):
paramsDict['Parameters'] = {}
paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
paramsDict['Parameters']['type'] = 'JDL'
arguments.append( ' -p ParametricParameters=%s' )
##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
#Add any JDL parameters to classad obeying lists with ';' rule
requirements = False
for name, props in paramsDict.items():
ptype = props['type']
value = props['value']
if name.lower() == 'requirements' and ptype == 'JDL':
self.log.verbose( 'Found existing requirements: %s' % ( value ) )
requirements = True
if re.search( '^JDL', ptype ):
if type( value ) == list:
if type( value[0] ) == list:
classadJob.insertAttributeVectorStringList( name, value )
else:
classadJob.insertAttributeVectorString( name, value )
elif value == "%s":
classadJob.insertAttributeInt( name, value )
elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
classadJob.insertAttributeString( name, value )
else:
classadJob.insertAttributeVectorString( name, value.split( ';' ) )
if not requirements:
reqtsDict = self.reqParams
exprn = ''
plus = ''
for name, props in paramsDict.items():
ptype = paramsDict[name]['type']
value = paramsDict[name]['value']
if not ptype == 'dict':
if ptype == 'JDLReqt':
if value and not value.lower() == 'any':
plus = ' && '
if re.search( ';', value ):
for val in value.split( ';' ):
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( val ) ) + plus
else:
exprn += reqtsDict[name].replace( 'NAME', name ).replace( 'VALUE', str( value ) ) + plus
if len( plus ):
exprn = exprn[:-len( plus )]
if not exprn:
exprn = 'true'
self.log.verbose( 'Requirements: %s' % ( exprn ) )
#classadJob.set_expression('Requirements', exprn)
self.addToInputSandbox.remove( scriptname )
self.addToOutputSandbox.remove( self.stdout )
self.addToOutputSandbox.remove( self.stderr )
jdl = classadJob.asJDL()
start = jdl.find( '[' )
end = jdl.rfind( ']' )
return jdl[( start + 1 ):( end - 1 )]
def getDestinationSEList(outputSE, site, outputmode='Any'):
""" Evaluate the output SE list from a workflow and return the concrete list
of SEs to upload output data.
"""
# Add output SE defined in the job description
gLogger.info('Resolving workflow output SE description: %s' % outputSE)
# Check if the SE is defined explicitly for the site
prefix = site.split('.')[0]
country = site.split('.')[-1]
# Concrete SE name
result = gConfig.getOptions('/Resources/StorageElements/' + outputSE)
if result['OK']:
gLogger.info('Found concrete SE %s' % outputSE)
return S_OK([outputSE])
# There is an alias defined for this Site
alias_se = gConfig.getValue('/Resources/Sites/%s/%s/AssociatedSEs/%s' % (prefix, site, outputSE), [])
if alias_se:
gLogger.info('Found associated SE for site %s' % (alias_se))
return S_OK(alias_se)
localSEs = getSEsForSite(site)['Value']
gLogger.verbose('Local SE list is: %s' % (localSEs))
groupSEs = gConfig.getValue('/Resources/StorageElementGroups/' + outputSE, [])
gLogger.verbose('Group SE list is: %s' % (groupSEs))
if not groupSEs:
return S_ERROR('Failed to resolve SE ' + outputSE)
if outputmode.lower() == "local":
for se in localSEs:
if se in groupSEs:
gLogger.info('Found eligible local SE: %s' % (se))
return S_OK([se])
#check if country is already one with associated SEs
associatedSE = gConfig.getValue('/Resources/Countries/%s/AssociatedSEs/%s' % (country, outputSE), '')
if associatedSE:
gLogger.info('Found associated SE %s in /Resources/Countries/%s/AssociatedSEs/%s' % (associatedSE, country, outputSE))
return S_OK([associatedSE])
# Final check for country associated SE
count = 0
assignedCountry = country
while count < 10:
gLogger.verbose('Loop count = %s' % (count))
gLogger.verbose("/Resources/Countries/%s/AssignedTo" % assignedCountry)
opt = gConfig.getOption("/Resources/Countries/%s/AssignedTo" % assignedCountry)
if opt['OK'] and opt['Value']:
assignedCountry = opt['Value']
gLogger.verbose('/Resources/Countries/%s/AssociatedSEs' % assignedCountry)
assocCheck = gConfig.getOption('/Resources/Countries/%s/AssociatedSEs' % assignedCountry)
if assocCheck['OK'] and assocCheck['Value']:
break
count += 1
if not assignedCountry:
return S_ERROR('Could not determine associated SE list for %s' % country)
alias_se = gConfig.getValue('/Resources/Countries/%s/AssociatedSEs/%s' % (assignedCountry, outputSE), [])
if alias_se:
gLogger.info('Found alias SE for site: %s' % alias_se)
return S_OK(alias_se)
else:
gLogger.error('Could not establish alias SE for country %s from section: /Resources/Countries/%s/AssociatedSEs/%s' % (country, assignedCountry, outputSE))
return S_ERROR('Failed to resolve SE ' + outputSE)
# For collective Any and All modes return the whole group
# Make sure that local SEs are passing first
newSEList = []
for se in groupSEs:
if se in localSEs:
newSEList.append(se)
uniqueSEs = uniqueElements(newSEList + groupSEs)
gLogger.verbose('Found unique SEs: %s' % (uniqueSEs))
return S_OK(uniqueSEs)
def _prepareRemoteHost(self, host=None):
"""Prepare remote directories and upload control script"""
ssh = SSH(host=host, parameters=self.ceParameters)
# Make remote directories
dirTuple = tuple(
uniqueElements([
self.sharedArea, self.executableArea, self.infoArea,
self.batchOutput, self.batchError, self.workArea
]))
nDirs = len(dirTuple)
cmd = "mkdir -p %s; " * nDirs % dirTuple
cmd = "bash -c '%s'" % cmd
self.log.verbose("Creating working directories on %s" %
self.ceParameters["SSHHost"])
result = ssh.sshCall(30, cmd)
if not result["OK"]:
self.log.error("Failed creating working directories",
"(%s)" % result["Message"][1])
return result
status, output, _error = result["Value"]
if status == -1:
self.log.error("Timeout while creating directories")
return S_ERROR(errno.ETIME, "Timeout while creating directories")
if "cannot" in output:
self.log.error("Failed to create directories", "(%s)" % output)
return S_ERROR(errno.EACCES, "Failed to create directories")
# Upload the control script now
result = self._generateControlScript()
if not result["OK"]:
self.log.warn("Failed generating control script")
return result
localScript = result["Value"]
self.log.verbose("Uploading %s script to %s" %
(self.batchSystem.__class__.__name__,
self.ceParameters["SSHHost"]))
remoteScript = "%s/execute_batch" % self.sharedArea
result = ssh.scpCall(30,
localScript,
remoteScript,
postUploadCommand="chmod +x %s" % remoteScript)
if not result["OK"]:
self.log.warn("Failed uploading control script: %s" %
result["Message"][1])
return result
status, output, _error = result["Value"]
if status != 0:
if status == -1:
self.log.warn("Timeout while uploading control script")
return S_ERROR("Timeout while uploading control script")
self.log.warn("Failed uploading control script: %s" % output)
return S_ERROR("Failed uploading control script")
# Delete the generated control script locally
try:
os.remove(localScript)
except OSError:
self.log.warn(
"Failed removing the generated control script locally")
return S_ERROR(
"Failed removing the generated control script locally")
# Chmod the control scripts
# self.log.verbose( 'Chmod +x control script' )
# result = ssh.sshCall( 10, "chmod +x %s/%s" % ( self.sharedArea, self.controlScript ) )
# if not result['OK']:
# self.log.warn( 'Failed chmod control script: %s' % result['Message'][1] )
# return result
# status, output, _error = result['Value']
# if status != 0:
# if status == -1:
# self.log.warn( 'Timeout while chmod control script' )
# return S_ERROR( 'Timeout while chmod control script' )
# else:
# self.log.warn( 'Failed uploading chmod script: %s' % output )
# return S_ERROR( 'Failed uploading chmod script' )
return S_OK()
elif tarfile.is_tarfile(fname):
try:
gLogger.debug('Unpacking catalog XML file %s' %
(os.path.join(directory, fname)))
tarFile = tarfile.open(os.path.join(directory, fname), 'r')
for member in tarFile.getmembers():
tarFile.extract(member, directory)
poolCatalogList.append(
os.path.join(directory, member.name))
except Exception, x:
gLogger.error('Could not untar %s with exception %s' %
(fname, str(x)))
else:
poolCatalogList.append(fname)
poolCatalogList = uniqueElements(poolCatalogList)
#Now have list of all XML files but some may not be Pool XML catalogs...
finalCatList = []
for possibleCat in poolCatalogList:
try:
cat = PoolXMLCatalog(possibleCat)
finalCatList.append(possibleCat)
except Exception, x:
gLogger.debug('Ignoring non-POOL catalogue file %s' % possibleCat)
gLogger.debug('Final list of catalog files are: %s' %
string.join(finalCatList, ', '))
return finalCatList
