def web_listPublicDesktopStates( self ):
up = self.__getUP()
result = yield self.threadTask( up.listAvailableVars )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
data = result['Value']
paramNames = ['UserName', 'Group', 'VO', 'desktop']
records = []
for i in data:
records += [dict( zip( paramNames, i ) )]
sharedDesktops = {}
for i in records:
result = yield self.threadTask( up.getVarPermissions, i['desktop'] )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
if result['Value']['ReadAccess'] == 'ALL':
print i['UserName'], i['Group'], i
result = yield self.threadTask( up.retrieveVarFromUser , i['UserName'], i['Group'], i['desktop'] )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
if i['UserName'] not in sharedDesktops:
sharedDesktops[i['UserName']] = {}
sharedDesktops[i['UserName']][i['desktop']] = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( result['Value'] ) ) )[0] )
sharedDesktops[i['UserName']]['Metadata'] = i
else:
sharedDesktops[i['UserName']][i['desktop']] = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( result['Value'] ) ) )[0] )
sharedDesktops[i['UserName']]['Metadata'] = i
self.finish( sharedDesktops )
def __call__(self):
"""execute RPC stub"""
# # decode arguments
try:
decode, length = DEncode.decode(self.operation.Arguments)
self.log.debug("decoded len=%s val=%s" % (length, decode))
except ValueError as error:
self.log.exception(error)
self.operation.Error = str(error)
self.operation.Status = "Failed"
return S_ERROR(str(error))
# Ensure the forwarded request is done on behalf of the request owner
decode[0][1]["delegatedDN"] = self.request.OwnerDN
decode[0][1]["delegatedGroup"] = self.request.OwnerGroup
# ForwardDiset is supposed to be used with a host certificate
useServerCertificate = gConfig.useServerCertificate()
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "true")
forward = executeRPCStub(decode)
if not useServerCertificate:
gConfigurationData.setOptionInCFG(
"/DIRAC/Security/UseServerCertificate", "false")
if not forward["OK"]:
self.log.error(
"unable to execute operation",
"'%s' : %s" % (self.operation.Type, forward["Message"]))
self.operation.Error = forward["Message"]
return forward
self.log.info("DISET forwarding done")
self.operation.Status = "Done"
return S_OK()
def __call__(self):
"""It expects to find the arguments for tc.setFileStatusForTransformation in operation.Arguments"""
try:
setFileStatusDict = DEncode.decode(self.operation.Arguments)[0]
self.log.debug("decoded filStatusDict=%s" % str(setFileStatusDict))
except ValueError as error:
self.log.exception(error)
self.operation.Error = str(error)
self.operation.Status = "Failed"
return S_ERROR(str(error))
tc = TransformationClient()
setStatus = tc.setFileStatusForTransformation(
setFileStatusDict["transformation"],
setFileStatusDict["statusDict"], setFileStatusDict["force"])
if not setStatus["OK"]:
errorStr = "failed to change status: %s" % setStatus["Message"]
self.operation.Error = errorStr
self.log.warn(errorStr)
return S_ERROR(self.operation.Error)
else:
self.operation.Status = "Done"
return S_OK()
def extractRequestFromFileId( fileId ):
stub = fileId[2:]
compressType = fileId[0]
if compressType == 'Z':
gLogger.info( "Compressed request, uncompressing" )
try:
stub = base64.urlsafe_b64decode( stub )
except Exception as e:
gLogger.error( "Oops! Plot request is not properly encoded!", str( e ) )
return S_ERROR( "Oops! Plot request is not properly encoded!: %s" % str( e ) )
try:
stub = zlib.decompress( stub )
except Exception as e:
gLogger.error( "Oops! Plot request is invalid!", str( e ) )
return S_ERROR( "Oops! Plot request is invalid!: %s" % str( e ) )
elif compressType == 'S':
gLogger.info( "Base64 request, decoding" )
try:
stub = base64.urlsafe_b64decode( stub )
except Exception as e:
gLogger.error( "Oops! Plot request is not properly encoded!", str( e ) )
return S_ERROR( "Oops! Plot request is not properly encoded!: %s" % str( e ) )
elif compressType == 'R':
#Do nothing, it's already uncompressed
pass
else:
gLogger.error( "Oops! Stub type is unknown", compressType )
return S_ERROR( "Oops! Stub type '%s' is unknown :P" % compressType )
plotRequest, stubLength = DEncode.decode( stub )
if len( stub ) != stubLength:
gLogger.error( "Oops! The stub is longer than the data :P" )
return S_ERROR( "Oops! The stub is longer than the data :P" )
return S_OK( plotRequest )
def printOperation(indexOperation, verbose=True, onlyFailed=False):
global output
i, op = indexOperation
prStr = ''
if op.SourceSE:
prStr += 'SourceSE: %s' % op.SourceSE
if op.TargetSE:
prStr += (' - ' if prStr else '') + 'TargetSE: %s' % op.TargetSE
if prStr:
prStr += ' - '
prStr += 'Created %s, Updated %s' % (op.CreationTime, op.LastUpdate)
if op.Type == 'ForwardDISET' and op.Arguments:
from DIRAC.Core.Utilities import DEncode
decode, _length = DEncode.decode(op.Arguments)
if verbose:
output = ''
prettyPrint(decode, offset=10)
prStr += '\n Arguments:\n' + output.strip('\n')
else:
prStr += '\n Service: %s' % decode[0][0]
gLogger.always(" [%s] Operation Type='%s' ID=%s Order=%s Status='%s'%s%s" %
(i, op.Type,
op.OperationID if hasattr(op, 'OperationID') else '(not set yet)',
op.Order, op.Status,
(" Error='%s'" % op.Error) if op.Error and op.Error.strip() else "",
(" Catalog=%s" % op.Catalog) if op.Catalog else ""))
if prStr:
gLogger.always(" %s" % prStr)
for indexFile in enumerate(op):
if not onlyFailed or indexFile[1].Status == 'Failed':
printFile(indexFile)
def extractRequestFromFileId(fileId):
stub = fileId[2:]
compressType = fileId[0]
if compressType == "Z":
gLogger.info("Compressed request, uncompressing")
try:
# Encoding is only required for Python 2 and can be removed when Python 2 support is no longer needed
stub = base64.urlsafe_b64decode(stub.encode())
except Exception as e:
gLogger.error("Oops! Plot request is not properly encoded!", str(e))
return S_ERROR("Oops! Plot request is not properly encoded!: %s" % str(e))
try:
stub = zlib.decompress(stub)
except Exception as e:
gLogger.error("Oops! Plot request is invalid!", str(e))
return S_ERROR("Oops! Plot request is invalid!: %s" % str(e))
elif compressType == "S":
gLogger.info("Base64 request, decoding")
try:
stub = base64.urlsafe_b64decode(stub)
except Exception as e:
gLogger.error("Oops! Plot request is not properly encoded!", str(e))
return S_ERROR("Oops! Plot request is not properly encoded!: %s" % str(e))
elif compressType == "R":
# Do nothing, it's already uncompressed
pass
else:
gLogger.error("Oops! Stub type is unknown", compressType)
return S_ERROR("Oops! Stub type '%s' is unknown :P" % compressType)
plotRequest, stubLength = DEncode.decode(stub)
if len(stub) != stubLength:
gLogger.error("Oops! The stub is longer than the data :P")
return S_ERROR("Oops! The stub is longer than the data :P")
return S_OK(plotRequest)
def getAlarmLog(self, alarmId):
try:
alarmId = int(alarmId)
except Exception:
return S_ERROR("Alarm id must be a non decimal number")
sqlSel = "SELECT %s FROM `ntf_AlarmLog` WHERE AlarmId=%d ORDER BY Timestamp ASC" % (
",".join(self.__alarmLogFields),
alarmId,
)
result = self._query(sqlSel)
if not result["OK"]:
return result
decodedRows = []
for row in result["Value"]:
decodedRows.append(list(row))
if not row[3]:
decodedRows.append(list(row))
continue
dec = DEncode.decode(row[3])
decodedRows[-1][3] = dec[0]
resultDict = {}
resultDict["ParameterNames"] = self.__alarmLogFields
resultDict["Records"] = decodedRows
return S_OK(resultDict)
def plotView(self, viewRequest):
"""
Generate all plots for a view
"""
views = self.__createCatalog().getViewById(viewRequest['id'])
if len(views) == 0:
return S_ERROR("View does not exist")
viewData = views[0]
viewDefinition = DEncode.decode(str(viewData[0]))[0]
neededVarFields = List.fromChar(viewData[1], ",")
if len(neededVarFields) > 0:
if 'varData' not in viewRequest:
return S_ERROR("Missing variable fields %s!" %
", ".join(neededVarFields))
missingVarFields = []
for neededField in neededVarFields:
if neededField in viewRequest['varData']:
viewDefinition['definition'][neededField] = viewRequest[
'varData'][neededField]
else:
missingVarFields.append(neededField)
if len(missingVarFields) > 0:
return S_ERROR("Missing required fields %s!" %
", ".join(missingVarFields))
return self.generatePlots(viewRequest['fromSecs'],
viewRequest['toSecs'], viewDefinition,
viewRequest['size'])
def printOperation( indexOperation, verbose = True, onlyFailed = False ):
global output
i, op = indexOperation
prStr = ''
if op.SourceSE:
prStr += 'SourceSE: %s' % op.SourceSE
if op.TargetSE:
prStr += ( ' - ' if prStr else '' ) + 'TargetSE: %s' % op.TargetSE
if prStr:
prStr += ' - '
prStr += 'Created %s, Updated %s' % ( op.CreationTime, op.LastUpdate )
if op.Type == 'ForwardDISET':
from DIRAC.Core.Utilities import DEncode
decode, _length = DEncode.decode( op.Arguments )
if verbose:
output = ''
prettyPrint( decode, offset = 10 )
prStr += '\n Arguments:\n' + output.strip( '\n' )
else:
prStr += '\n Service: %s' % decode[0][0]
gLogger.always( " [%s] Operation Type='%s' ID=%s Order=%s Status='%s'%s%s" % ( i, op.Type, op.OperationID,
op.Order, op.Status,
( " Error='%s'" % op.Error ) if op.Error and op.Error.strip() else "",
( " Catalog=%s" % op.Catalog ) if op.Catalog else "" ) )
if prStr:
gLogger.always( " %s" % prStr )
for indexFile in enumerate( op ):
if not onlyFailed or indexFile[1].Status == 'Failed':
printFile( indexFile )
def getFileInfo(lfn):
""" Retrieve the file info
"""
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Core.Utilities import DEncode
from DIRAC import gLogger
fc = FileCatalogClient()
lumi = 0
nbevts = 0
res = fc.getFileUserMetadata(lfn)
if not res['OK']:
gLogger.error("Failed to get metadata of %s" % lfn)
return (0,0,{})
if res['Value'].has_key('Luminosity'):
lumi += float(res['Value']['Luminosity'])
addinfo = {}
if 'AdditionalInfo' in res['Value']:
addinfo = res['Value']['AdditionalInfo']
if addinfo.count("{"):
addinfo = eval(addinfo)
else:
addinfo = DEncode.decode(addinfo)[0]
if "NumberOfEvents" in res['Value'].keys():
nbevts += int(res['Value']['NumberOfEvents'])
return (float(lumi),int(nbevts),addinfo)
def _getFileInfo(lfn):
""" Retrieve the file info
"""
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Core.Utilities import DEncode
from DIRAC import gLogger
fc = FileCatalogClient()
lumi = 0
nbevts = 0
res = fc.getFileUserMetadata(lfn)
if not res['OK']:
gLogger.error("Failed to get metadata of %s" % lfn)
return (0, 0, {})
if 'Luminosity' in res['Value']:
lumi += float(res['Value']['Luminosity'])
addinfo = {}
if 'AdditionalInfo' in res['Value']:
addinfo = res['Value']['AdditionalInfo']
if addinfo.count("{"):
addinfo = eval(addinfo)
else:
addinfo = DEncode.decode(addinfo)[0]
if "NumberOfEvents" in res['Value'].keys():
nbevts += int(res['Value']['NumberOfEvents'])
return (float(lumi), int(nbevts), addinfo)
def plotView(self, viewRequest):
"""
Generates all plots for a view.
:type viewRequest: dictionary
:param viewRequest: Containing information of the view like 'id', 'fromSecs', 'toSecs', etc.
:return: S_OK with a list of files.
"""
views = self.__createCatalog().getViewById(viewRequest['id'])
if len(views) == 0:
return S_ERROR("View does not exist")
viewData = views[0]
viewDefinition = DEncode.decode(str(viewData[0]))[0]
neededVarFields = List.fromChar(viewData[1], ",")
if len(neededVarFields) > 0:
if 'varData' not in viewRequest:
return S_ERROR("Missing variable fields %s!" %
", ".join(neededVarFields))
missingVarFields = []
for neededField in neededVarFields:
if neededField in viewRequest['varData']:
viewDefinition['definition'][neededField] = viewRequest[
'varData'][neededField]
else:
missingVarFields.append(neededField)
if len(missingVarFields) > 0:
return S_ERROR("Missing required fields %s!" %
", ".join(missingVarFields))
return self.generatePlots(viewRequest['fromSecs'],
viewRequest['toSecs'], viewDefinition,
viewRequest['size'])
def receiveData( self, maxBufferSize = 0, blockAfterKeepAlive = True, idleReceive = False ):
self.__updateLastActionTimestamp()
if self.receivedMessages:
return self.receivedMessages.pop( 0 )
#Buffer size can't be less than 0
maxBufferSize = max( maxBufferSize, 0 )
try:
#Look either for message length of keep alive magic string
iSeparatorPosition = self.byteStream.find( ":", 0, 10 )
keepAliveMagicLen = len( BaseTransport.keepAliveMagic )
isKeepAlive = self.byteStream.find( BaseTransport.keepAliveMagic, 0, keepAliveMagicLen ) == 0
#While not found the message length or the ka, keep receiving
while iSeparatorPosition == -1 and not isKeepAlive:
retVal = self._read( 1024 )
#If error return
if not retVal[ 'OK' ]:
return retVal
#If closed return error
if not retVal[ 'Value' ]:
return S_ERROR( "Peer closed connection" )
#New data!
self.byteStream += retVal[ 'Value' ]
#Look again for either message length of ka magic string
iSeparatorPosition = self.byteStream.find( ":", 0, 10 )
isKeepAlive = self.byteStream.find( BaseTransport.keepAliveMagic, 0, keepAliveMagicLen ) == 0
#Over the limit?
if maxBufferSize and len( self.byteStream ) > maxBufferSize and iSeparatorPosition == -1 :
return S_ERROR( "Read limit exceeded (%s chars)" % maxBufferSize )
#Keep alive magic!
if isKeepAlive:
gLogger.debug( "Received keep alive header" )
#Remove the ka magic from the buffer and process the keep alive
self.byteStream = self.byteStream[ keepAliveMagicLen: ]
return self.__processKeepAlive( maxBufferSize, blockAfterKeepAlive )
#From here it must be a real message!
#Process the size and remove the msg length from the bytestream
size = int( self.byteStream[ :iSeparatorPosition ] )
self.byteStream = self.byteStream[ iSeparatorPosition + 1: ]
#Receive while there's still data to be received
while len( self.byteStream ) < size:
retVal = self._read( size - len( self.byteStream ), skipReadyCheck = True )
if not retVal[ 'OK' ]:
return retVal
if not retVal[ 'Value' ]:
return S_ERROR( "Peer closed connection" )
self.byteStream += retVal[ 'Value' ]
if maxBufferSize and len( self.byteStream ) > maxBufferSize:
return S_ERROR( "Read limit exceeded (%s chars)" % maxBufferSize )
#Data is here! take it out from the bytestream, dencode and return
data = self.byteStream[ :size ]
self.byteStream = self.byteStream[ size: ]
try:
data = DEncode.decode( data )[0]
except Exception, e:
return S_ERROR( "Could not decode received data: %s" % str( e ) )
if idleReceive:
self.receivedMessages.append( data )
return S_OK()
return data
def receiveData( self, maxBufferSize = 0, blockAfterKeepAlive = True, idleReceive = False ):
self.__updateLastActionTimestamp()
if self.receivedMessages:
return self.receivedMessages.pop( 0 )
#Buffer size can't be less than 0
maxBufferSize = max( maxBufferSize, 0 )
try:
#Look either for message length of keep alive magic string
iSeparatorPosition = self.byteStream.find( ":", 0, 10 )
keepAliveMagicLen = len( BaseTransport.keepAliveMagic )
isKeepAlive = self.byteStream.find( BaseTransport.keepAliveMagic, 0, keepAliveMagicLen ) == 0
#While not found the message length or the ka, keep receiving
while iSeparatorPosition == -1 and not isKeepAlive:
retVal = self._read( 1024 )
#If error return
if not retVal[ 'OK' ]:
return retVal
#If closed return error
if not retVal[ 'Value' ]:
return S_ERROR( "Peer closed connection" )
#New data!
self.byteStream += retVal[ 'Value' ]
#Look again for either message length of ka magic string
iSeparatorPosition = self.byteStream.find( ":", 0, 10 )
isKeepAlive = self.byteStream.find( BaseTransport.keepAliveMagic, 0, keepAliveMagicLen ) == 0
#Over the limit?
if maxBufferSize and len( self.byteStream ) > maxBufferSize and iSeparatorPosition == -1 :
return S_ERROR( "Read limit exceeded (%s chars)" % maxBufferSize )
#Keep alive magic!
if isKeepAlive:
gLogger.debug( "Received keep alive header" )
#Remove the ka magic from the buffer and process the keep alive
self.byteStream = self.byteStream[ keepAliveMagicLen: ]
return self.__processKeepAlive( maxBufferSize, blockAfterKeepAlive )
#From here it must be a real message!
#Process the size and remove the msg length from the bytestream
size = int( self.byteStream[ :iSeparatorPosition ] )
self.byteStream = self.byteStream[ iSeparatorPosition + 1: ]
#Receive while there's still data to be received
while len( self.byteStream ) < size:
retVal = self._read( size - len( self.byteStream ), skipReadyCheck = True )
if not retVal[ 'OK' ]:
return retVal
if not retVal[ 'Value' ]:
return S_ERROR( "Peer closed connection" )
self.byteStream += retVal[ 'Value' ]
if maxBufferSize and len( self.byteStream ) > maxBufferSize:
return S_ERROR( "Read limit exceeded (%s chars)" % maxBufferSize )
#Data is here! take it out from the bytestream, dencode and return
data = self.byteStream[ :size ]
self.byteStream = self.byteStream[ size: ]
try:
data = DEncode.decode( data )[0]
except Exception, e:
return S_ERROR( "Could not decode received data: %s" % str( e ) )
if idleReceive:
self.receivedMessages.append( data )
return S_OK()
return data
def export_tryView( self, fromSecs, toSecs, viewDescriptionStub ):
"""
Generates plots based on a DEncoded view description
"""
viewDescription, _stubLength = DEncode.decode( viewDescriptionStub )
if not 'definition' in viewDescription:
return S_ERROR( "No plot definition given" )
defDict = viewDescription[ 'definition' ]
defDict[ 'sources.setup' ] = self.serviceInfoDict[ 'clientSetup' ]
return gServiceInterface.generatePlots( fromSecs, toSecs, viewDescription )
def __decodeVar(self, data, dataTypeRE):
try:
dataObj, lenData = DEncode.decode(data)
except Exception as e:
return S_ERROR("Cannot decode data: %s" % str(e))
if dataTypeRE:
result = self.checkTypeRe(dataObj, dataTypeRE)
if not result['OK']:
return result
return S_OK(dataObj)
def __decodeVar( self, data, dataTypeRE ):
try:
dataObj, lenData = DEncode.decode( data )
except Exception as e:
return S_ERROR( "Cannot decode data: %s" % str( e ) )
if dataTypeRE:
result = self.checkTypeRe( dataObj, dataTypeRE )
if not result[ 'OK' ]:
return result
return S_OK( dataObj )
def web_listAppState( self ):
up = self.__getUP()
result = yield self.threadTask( up.retrieveAllVars )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
data = result[ 'Value' ]
for k in data:
#Unpack data
data[ k ] = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( data[ k ] ) ) )[0] )
self.finish( data )
def decode(encodedData):
""" Decode the encoded string
:param encodedData: encoded string
:return: the decoded objects, encoded object length
"""
if os.getenv('DIRAC_USE_JSON_DECODE', 'NO').lower() in ('yes', 'true'):
try:
# 'null' is a special case.
# None is encoded as 'null' as JSON
# that DEncode would understand as None, since it starts with 'n'
# but the length of the decoded string will not be correct
if encodedData == 'null':
raise Exception
return DEncode.decode(encodedData)
except Exception:
return JEncode.decode(encodedData)
return DEncode.decode(encodedData)
def __call__(self):
""" It expects to find the arguments for tc.setFileStatusForTransformation in operation.Arguments
"""
try:
setFileStatusDict = DEncode.decode(self.operation.Arguments)[0]
self.log.debug("decoded filStatusDict=%s" % str(setFileStatusDict))
except ValueError, error:
self.log.exception(error)
self.operation.Error = str(error)
self.operation.Status = "Failed"
return S_ERROR(str(error))
def web_listAppState(self):
up = self.__getUP()
result = yield self.threadTask(up.retrieveAllVars)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
for k in data:
#Unpack data
data[k] = json.loads(
DEncode.decode(zlib.decompress(base64.b64decode(data[k])))[0])
self.finish(data)
def __call__( self ):
""" It expects to find the arguments for tc.setFileStatusForTransformation in operation.Arguments
"""
try:
setFileStatusDict = DEncode.decode( self.operation.Arguments )[0]
self.log.debug( "decoded filStatusDict=%s" % str( setFileStatusDict ) )
except ValueError, error:
self.log.exception( error )
self.operation.Error = str( error )
self.operation.Status = "Failed"
return S_ERROR( str( error ) )
def __call__( self ):
""" execute RPC stub """
# # decode arguments
try:
decode, length = DEncode.decode( self.operation.Arguments )
self.log.debug( "decoded len=%s val=%s" % ( length, decode ) )
except ValueError, error:
self.log.exception( error )
self.operation.Error = str( error )
self.operation.Status = "Failed"
return S_ERROR( str( error ) )
def retrieveAllVars(self):
rpcClient = self.__getRPCClient()
result = rpcClient.retrieveProfileAllVars(self.profile)
if not result["OK"]:
return result
try:
encodedData = result["Value"]
dataObj = {key: DEncode.decode(value.encode())[0] for key, value in encodedData.items()}
except Exception as e:
return S_ERROR("Cannot decode data: %s" % str(e))
return S_OK(dataObj)
def export_saveView(self, viewName, viewDescriptionStub):
"""
Saves a view
"""
if not viewName:
return S_OK("View name not valid")
viewDescription, _stubLength = DEncode.decode(viewDescriptionStub)
if "definition" not in viewDescription:
return S_ERROR("No plot definition given")
defDict = viewDescription["definition"]
defDict["sources.setup"] = self.serviceInfoDict["clientSetup"]
return gServiceInterface.saveView(viewName, viewDescription)
def web_loadAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
result = yield self.threadTask(up.retrieveVar, name)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
data, count = DEncode.decode(zlib.decompress(base64.b64decode(data)))
self.finish(data)
def pythonCall(self, function, *stArgs, **stKeyArgs):
""" call python function :function: with :stArgs: and :stKeyArgs: """
from DIRAC.Core.Utilities import DEncode
self.log.verbose('pythonCall:', function.__name__)
readFD, writeFD = os.pipe()
pid = os.fork()
self.childPID = pid
if pid == 0:
os.close(readFD)
self.__executePythonFunction(function, writeFD, *stArgs,
**stKeyArgs)
# FIXME: the close it is done at __executePythonFunction, do we need it here?
os.close(writeFD)
else:
os.close(writeFD)
readSeq = self.__selectFD([readFD])
if readSeq == False:
return S_ERROR("Can't read from call %s" % (function.__name__))
try:
if len(readSeq) == 0:
self.log.debug('Timeout limit reached for pythonCall',
function.__name__)
self.__killPid(pid)
#HACK to avoid python bug
# self.wait()
retries = 10000
while os.waitpid(pid, 0) == -1 and retries > 0:
time.sleep(0.001)
retries -= 1
return S_ERROR('%d seconds timeout for "%s" call' %
(self.timeout, function.__name__))
elif readSeq[0] == readFD:
retDict = self.__readFromFD(readFD)
os.waitpid(pid, 0)
if retDict['OK']:
dataStub = retDict['Value']
if not dataStub:
return S_ERROR(
"Error decoding data coming from call")
retObj, stubLen = DEncode.decode(dataStub)
if stubLen == len(dataStub):
return retObj
else:
return S_ERROR(
"Error decoding data coming from call")
return retDict
finally:
os.close(readFD)
def web_loadAppState( self ):
up = self.__getUP()
try:
name = self.request.arguments[ 'name' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
result = yield self.threadTask( up.retrieveVar, name )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result)
data = result[ 'Value' ]
data, count = DEncode.decode( zlib.decompress( base64.b64decode( data ) ) )
self.finish( data )
def export_saveView( self, viewName, viewDescriptionStub ):
"""
Saves a view
"""
if len( viewName ) == 0:
return S_OK( "View name not valid" )
viewDescription, _stubLength = DEncode.decode( viewDescriptionStub )
if not 'definition' in viewDescription:
return S_ERROR( "No plot definition given" )
defDict = viewDescription[ 'definition' ]
defDict[ 'sources.setup' ] = self.serviceInfoDict[ 'clientSetup' ]
return gServiceInterface.saveView( viewName, viewDescription )
def retrieveAllVars(self):
rpcClient = self.__getRPCClient()
result = rpcClient.retrieveProfileAllVars(self.profile)
if not result['OK']:
return result
try:
encodedData = result['Value']
dataObj = {}
for k in encodedData:
v, lenData = DEncode.decode(encodedData[k])
dataObj[k] = v
except Exception, e:
return S_ERROR("Cannot decode data: %s" % str(e))
def retrieveAllVars( self ):
rpcClient = self.__getRPCClient()
result = rpcClient.retrieveProfileAllVars( self.profile )
if not result[ 'OK' ]:
return result
try:
encodedData = result[ 'Value' ]
dataObj = {}
for k in encodedData:
v, lenData = DEncode.decode( encodedData[k] )
dataObj[ k ] = v
except Exception, e:
return S_ERROR( "Cannot decode data: %s" % str( e ) )
def __call__( self ):
""" execute RPC stub """
# # decode arguments
try:
decode, length = DEncode.decode( self.operation.Arguments )
# FIXME: to be removed when the problem is fixed in the LHCb HLT farm
if 'CPUFactor_to_be_replace' in self.operation.Arguments and len( decode ) == 3 and type( decode[2] ) == type( tuple() ):
decode = ( decode[0], decode[1], tuple( [x.replace( 'CPUFactor_to_be_replace', '0' ) for x in decode[2]] ) )
self.log.debug( "decoded len=%s val=%s" % ( length, decode ) )
except ValueError, error:
self.log.exception( error )
self.operation.Error = str( error )
self.operation.Status = "Failed"
return S_ERROR( str( error ) )
def retrieveOptimizerParam(self, name):
if not self.__jobData.jobState:
return S_ERROR("This function can only be called inside the optimizeJob function")
result = self.__jobData.jobState.getOptParameter(name)
if not result['OK']:
return result
valenc = result['Value']
try:
value, encLength = DEncode.decode(valenc)
if encLength == len(valenc):
return S_OK(value)
except Exception:
self.jobLog.warn("Opt param %s doesn't seem to be dencoded %s" % (name, valenc))
return S_OK(eval(valenc))
def retrieveOptimizerParam(self, name):
if not self.__jobData.jobState:
return S_ERROR("This function can only be called inside the optimizeJob function")
result = self.__jobData.jobState.getOptParameter(name)
if not result['OK']:
return result
valenc = result['Value']
try:
value, encLength = DEncode.decode(valenc)
if encLength == len(valenc):
return S_OK(value)
except Exception:
self.jobLog.warn("Opt param %s doesn't seem to be dencoded %s" % (name, valenc))
return S_OK(eval(valenc))
def web_listPublicDesktopStates(self):
up = self.__getUP()
result = yield self.threadTask(up.listAvailableVars)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
paramNames = ['UserName', 'Group', 'VO', 'desktop']
records = []
for i in data:
records += [dict(zip(paramNames, i))]
sharedDesktops = {}
for i in records:
result = yield self.threadTask(up.getVarPermissions, i['desktop'])
if not result['OK']:
raise WErr.fromSERROR(result)
if result['Value']['ReadAccess'] == 'ALL':
print i['UserName'], i['Group'], i
result = yield self.threadTask(up.retrieveVarFromUser,
i['UserName'], i['Group'],
i['desktop'])
if not result['OK']:
raise WErr.fromSERROR(result)
if i['UserName'] not in sharedDesktops:
sharedDesktops[i['UserName']] = {}
sharedDesktops[i['UserName']][i['desktop']] = json.loads(
DEncode.decode(
zlib.decompress(base64.b64decode(
result['Value'])))[0])
sharedDesktops[i['UserName']]['Metadata'] = i
else:
sharedDesktops[i['UserName']][i['desktop']] = json.loads(
DEncode.decode(
zlib.decompress(base64.b64decode(
result['Value'])))[0])
sharedDesktops[i['UserName']]['Metadata'] = i
self.finish(sharedDesktops)
def __call__(self):
""" It expects to find the reqID in operation.Arguments
"""
try:
decode = DEncode.decode(self.operation.Arguments)
self.log.debug(decode)
gLogger.debug("Validating output")
except ValueError as error:
self.log.exception(error)
self.operation.Error = str(error)
self.operation.Status = "Failed"
return S_ERROR(str(error))
self.operation.Status = "Done"
return S_OK()
def web_loadAppState( self ):
self.__tc.setSetup( False )
try:
app = self.request.arguments[ 'app' ][-1]
name = self.request.arguments[ 'name' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
up = UserProfileClient( "Web/App/%s" % app )
result = yield self.threadTask( up.retrieveVar, name )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result)
data = result[ 'Value' ]
data, count = DEncode.decode( zlib.decompress( base64.b64decode( data ) ) )
self.set_header( "Content-Type", "application/json" )
self.finish( data )
def web_loadAppState(self):
self.__tc.setSetup(False)
try:
app = self.request.arguments['app'][-1]
name = self.request.arguments['name'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
up = UserProfileClient("Web/App/%s" % app)
result = yield self.threadTask(up.retrieveVar, name)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
data, count = DEncode.decode(zlib.decompress(base64.b64decode(data)))
self.set_header("Content-Type", "application/json")
self.finish(data)
def pythonCall( self, function, *stArgs, **stKeyArgs ):
""" call python function :function: with :stArgs: and :stKeyArgs: """
from DIRAC.Core.Utilities import DEncode
self.log.verbose( 'pythonCall:', function.__name__ )
readFD, writeFD = os.pipe()
pid = os.fork()
self.childPID = pid
if pid == 0:
os.close( readFD )
self.__executePythonFunction( function, writeFD, *stArgs, **stKeyArgs )
# FIXME: the close it is done at __executePythonFunction, do we need it here?
os.close( writeFD )
else:
os.close( writeFD )
readSeq = self.__selectFD( [ readFD ] )
if readSeq == False:
return S_ERROR( "Can't read from call %s" % ( function.__name__ ) )
try:
if len( readSeq ) == 0:
self.log.debug( 'Timeout limit reached for pythonCall', function.__name__ )
self.__killPid( pid )
#HACK to avoid python bug
# self.wait()
retries = 10000
while os.waitpid( pid, 0 ) == -1 and retries > 0:
time.sleep( 0.001 )
retries -= 1
return S_ERROR( '%d seconds timeout for "%s" call' % ( self.timeout, function.__name__ ) )
elif readSeq[0] == readFD:
retDict = self.__readFromFD( readFD )
os.waitpid( pid, 0 )
if retDict[ 'OK' ]:
dataStub = retDict[ 'Value' ]
if not dataStub:
return S_ERROR( "Error decoding data coming from call" )
retObj, stubLen = DEncode.decode( dataStub )
if stubLen == len( dataStub ):
return retObj
else:
return S_ERROR( "Error decoding data coming from call" )
return retDict
finally:
os.close( readFD )
def _run(self):
"""Execute the download and tarring."""
self.parameterDict = DEncode.decode(self.operation.Arguments)[0] # tuple: dict, number of characters
self.cacheFolder = os.path.join(self.cacheFolder, self.request.RequestName)
self._checkArchiveLFN()
for parameter, value in self.parameterDict.iteritems():
self.log.info('Parameters: %s = %s' % (parameter, value))
self.log.info('Cache folder: %r' % self.cacheFolder)
self.waitingFiles = self.getWaitingFilesList()
self.lfns = [opFile.LFN for opFile in self.waitingFiles]
self._checkReplicas()
self._downloadFiles()
self._tarFiles()
self._uploadTarBall()
self._registerDescendent()
self._markFilesDone()
def deserialize(stub):
dataTuple, _slen = DEncode.decode(stub)
if len(dataTuple) != 7:
return S_ERROR("Invalid stub")
#jid
if type(dataTuple[0]) not in (types.IntType, types.LongType):
return S_ERROR("Invalid stub 0")
#cache
if type(dataTuple[1]) != types.DictType:
return S_ERROR("Invalid stub 1")
#trace
if type(dataTuple[2]) != types.ListType:
return S_ERROR("Invalid stub 2")
#manifest
tdt3 = type(dataTuple[3])
if tdt3 != types.NoneType and (tdt3 != types.TupleType
and len(dataTuple[3]) != 2):
return S_ERROR("Invalid stub 3")
#initstate
if type(dataTuple[4]) != types.DictType:
return S_ERROR("Invalid stub 4")
#Insert into TQ
if type(dataTuple[5]) != types.BooleanType:
return S_ERROR("Invalid stub 5")
#Dirty Keys
if type(dataTuple[6]) != types.TupleType:
return S_ERROR("Invalid stub 6")
cjs = CachedJobState(dataTuple[0], skipInitState=True)
cjs.__cache = dataTuple[1]
cjs.__jobLog = dataTuple[2]
dt3 = dataTuple[3]
if dataTuple[3]:
manifest = JobManifest()
result = manifest.loadCFG(dt3[0])
if not result['OK']:
return result
if dt3[1]:
manifest.setDirty()
else:
manifest.clearDirty()
cjs.__manifest = manifest
cjs.__initState = dataTuple[4]
cjs.__insertIntoTQ = dataTuple[5]
cjs.__dirtyKeys = set(dataTuple[6])
return S_OK(cjs)
def deserialize(stub):
dataTuple, _slen = DEncode.decode(stub.encode())
if len(dataTuple) != 7:
return S_ERROR("Invalid stub")
# jid
if not isinstance(dataTuple[0], int):
return S_ERROR("Invalid stub 0")
# cache
if not isinstance(dataTuple[1], dict):
return S_ERROR("Invalid stub 1")
# trace
if not isinstance(dataTuple[2], list):
return S_ERROR("Invalid stub 2")
# manifest
if dataTuple[3] is not None and (
not isinstance(dataTuple[3],
(tuple, list)) and len(dataTuple[3]) != 2):
return S_ERROR("Invalid stub 3")
# initstate
if not isinstance(dataTuple[4], dict):
return S_ERROR("Invalid stub 4")
# Insert into TQ
if not isinstance(dataTuple[5], bool):
return S_ERROR("Invalid stub 5")
# Dirty Keys
if not isinstance(dataTuple[6], (tuple, list)):
return S_ERROR("Invalid stub 6")
cjs = CachedJobState(dataTuple[0], skipInitState=True)
cjs.__cache = dataTuple[1]
cjs.__jobLog = dataTuple[2]
dt3 = dataTuple[3]
if dataTuple[3]:
manifest = JobManifest()
result = manifest.loadCFG(dt3[0])
if not result["OK"]:
return result
if dt3[1]:
manifest.setDirty()
else:
manifest.clearDirty()
cjs.__manifest = manifest
cjs.__initState = dataTuple[4]
cjs.__insertIntoTQ = dataTuple[5]
cjs.__dirtyKeys = set(dataTuple[6])
return S_OK(cjs)
def deserialize(stub):
dataTuple, slen = DEncode.decode(stub)
if len(dataTuple) != 7:
return S_ERROR("Invalid stub")
# jid
if type(dataTuple[0]) not in (types.IntType, types.LongType):
return S_ERROR("Invalid stub 0")
# cache
if type(dataTuple[1]) != types.DictType:
return S_ERROR("Invalid stub 1")
# trace
if type(dataTuple[2]) != types.ListType:
return S_ERROR("Invalid stub 2")
# manifest
tdt3 = type(dataTuple[3])
if tdt3 != types.NoneType and (tdt3 != types.TupleType and len(dataTuple[3]) != 2):
return S_ERROR("Invalid stub 3")
# initstate
if type(dataTuple[4]) != types.DictType:
return S_ERROR("Invalid stub 4")
# Insert into TQ
if type(dataTuple[5]) != types.BooleanType:
return S_ERROR("Invalid stub 5")
# Dirty Keys
if type(dataTuple[6]) != types.TupleType:
return S_ERROR("Invalid stub 6")
cjs = CachedJobState(dataTuple[0], skipInitState=True)
cjs.__cache = dataTuple[1]
cjs.__jobLog = dataTuple[2]
dt3 = dataTuple[3]
if dataTuple[3]:
manifest = JobManifest()
result = manifest.loadCFG(dt3[0])
if not result["OK"]:
return result
if dt3[1]:
manifest.setDirty()
else:
manifest.clearDirty()
cjs.__manifest = manifest
cjs.__initState = dataTuple[4]
cjs.__insertIntoTQ = dataTuple[5]
cjs.__dirtyKeys = set(dataTuple[6])
return S_OK(cjs)
def deserialize(stub):
dataTuple, _slen = DEncode.decode(stub)
if len(dataTuple) != 7:
return S_ERROR("Invalid stub")
# jid
if not isinstance(dataTuple[0], (int, long)):
return S_ERROR("Invalid stub 0")
# cache
if not isinstance(dataTuple[1], dict):
return S_ERROR("Invalid stub 1")
# trace
if not isinstance(dataTuple[2], list):
return S_ERROR("Invalid stub 2")
# manifest
if dataTuple[3] is not None and (not isinstance(dataTuple[3], tuple) and len(dataTuple[3]) != 2):
return S_ERROR("Invalid stub 3")
# initstate
if not isinstance(dataTuple[4], dict):
return S_ERROR("Invalid stub 4")
# Insert into TQ
if not isinstance(dataTuple[5], bool):
return S_ERROR("Invalid stub 5")
# Dirty Keys
if not isinstance(dataTuple[6], tuple):
return S_ERROR("Invalid stub 6")
cjs = CachedJobState(dataTuple[0], skipInitState=True)
cjs.__cache = dataTuple[1]
cjs.__jobLog = dataTuple[2]
dt3 = dataTuple[3]
if dataTuple[3]:
manifest = JobManifest()
result = manifest.loadCFG(dt3[0])
if not result['OK']:
return result
if dt3[1]:
manifest.setDirty()
else:
manifest.clearDirty()
cjs.__manifest = manifest
cjs.__initState = dataTuple[4]
cjs.__insertIntoTQ = dataTuple[5]
cjs.__dirtyKeys = set(dataTuple[6])
return S_OK(cjs)
def web_changeView( self ):
up = self.__getUP()
try:
desktopName = self.request.arguments[ 'desktop' ][-1]
view = self.request.arguments[ 'view' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
result = yield self.threadTask( up.retrieveVar, desktopName )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
data = result['Value']
oDesktop = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( data ) ) )[0] )
oDesktop[unicode( 'view' )] = unicode( view )
oDesktop = json.dumps( oDesktop )
data = base64.b64encode( zlib.compress( DEncode.encode( oDesktop ), 9 ) )
result = yield self.threadTask( up.storeVar, desktopName, data )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
self.set_status( 200 )
self.finish()
def web_changeView(self):
up = self.__getUP()
try:
desktopName = self.request.arguments['desktop'][-1]
view = self.request.arguments['view'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
result = yield self.threadTask(up.retrieveVar, desktopName)
if not result['OK']:
raise WErr.fromSERROR(result)
data = result['Value']
oDesktop = json.loads(
DEncode.decode(zlib.decompress(base64.b64decode(data)))[0])
oDesktop[unicode('view')] = unicode(view)
oDesktop = json.dumps(oDesktop)
data = base64.b64encode(zlib.compress(DEncode.encode(oDesktop), 9))
result = yield self.threadTask(up.storeVar, desktopName, data)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
def plotView( self, viewRequest ):
"""
Generate all plots for a view
"""
views = self.__createCatalog().getViewById( viewRequest[ 'id' ] )
if len( views ) == 0:
return S_ERROR( "View does not exist" )
viewData = views[0]
viewDefinition = DEncode.decode( str( viewData[ 0 ] ) )[0]
neededVarFields = List.fromChar( viewData[1], "," )
if len( neededVarFields ) > 0:
if not 'varData' in viewRequest:
return S_ERROR( "Missing variable fields %s!" % ", ".join( neededVarFields ) )
missingVarFields = []
for neededField in neededVarFields:
if neededField in viewRequest[ 'varData' ]:
viewDefinition[ 'definition' ][ neededField ] = viewRequest[ 'varData' ][ neededField ]
else:
missingVarFields.append( neededField )
if len( missingVarFields ) > 0:
return S_ERROR( "Missing required fields %s!" % ", ".join( missingVarFields ) )
return self.generatePlots( viewRequest[ 'fromSecs' ], viewRequest[ 'toSecs' ], viewDefinition, viewRequest[ 'size' ] )
def getAlarmLog( self, alarmId ):
try:
alarmId = int( alarmId )
except:
return S_ERROR( "Alarm id must be a non decimal number" )
sqlSel = "SELECT %s FROM `ntf_AlarmLog` WHERE AlarmId=%d ORDER BY Timestamp ASC" % ( ",".join( self.__alarmLogFields ),
alarmId )
result = self._query( sqlSel )
if not result[ 'OK' ]:
return result
decodedRows = []
for row in result[ 'Value' ]:
decodedRows.append( list( row ) )
if not row[3]:
decodedRows.append( list( row ) )
continue
dec = DEncode.decode( row[ 3 ] )
decodedRows[-1][3] = dec[0]
resultDict = {}
resultDict['ParameterNames'] = self.__alarmLogFields
resultDict['Records'] = decodedRows
return S_OK( resultDict )
def __listFromXML( self, dom ):
resultList = []
"""
for child in dom.childNodes:
if child.nodeType == child.ELEMENT_NODE:
dname = child.nodeName
dom_dict = dom.getElementsByTagName(dname)[0]
if dom_dict.getAttribute('element_type') == 'dictionary':
ddict = self.__dictionaryFromXML(child)
resultList.append(ddict)
elif dom_dict.getAttribute('element_type') == 'list':
resultList = self.__listFromXML(child)
elif dom_dict.getAttribute('element_type') == 'leaf':
value = self.__getCharacterData(child)
resultList.append(value)
"""
for child in dom.childNodes:
if child.nodeType == child.ELEMENT_NODE:
dname = child.nodeName
dom_dict = dom.getElementsByTagName( dname )[0]
if dom_dict.getAttribute( 'element_type' ) == 'leaf':
value = self.__getCharacterData( child )
resultList, ignored = DEncode.decode( value )
return resultList
def exec_deserializeTask(cls, taskStub):
gLogger.info("DESERIALIZE %s" % taskStub)
return S_OK(DEncode.decode(taskStub)[0])
def createFileInfo(fmeta):
from DIRAC.Core.Utilities import DEncode
if 'ProdID' in fmeta:
del fmeta['ProdID']
info = []
info.append(" - Machine %s" % fmeta['Machine'])
del fmeta['Machine']
info.append(" - Energy %sGeV"% fmeta['Energy'])
del fmeta['Energy']
if 'MachineParams' in fmeta:
info.append(' - The machine parameters are %s' % fmeta['MachineParams'])
del fmeta['MachineParams']
if 'EvtClass' in fmeta:
info.append(' - Is among the %s event class' % fmeta['EvtClass'])
del fmeta['EvtClass']
if 'ProcessID' in fmeta:
info.append(' - Is the ProcessID %s' % str(fmeta['ProcessID']))
del fmeta['ProcessID']
elif 'GenProcessID' in fmeta:
info.append(' - Is the GenProcessID %s' % str(fmeta['GenProcessID']))
del fmeta['GenProcessID']
info.append(" - Is the %s event type" % fmeta["EvtType"])
del fmeta["EvtType"]
if 'Polarisation' in fmeta:
info.append(" - Has %s polarisation" % fmeta['Polarisation'])
del fmeta["Polarisation"]
if 'BeamParticle1' in fmeta:
info.append(" - Beam 1 particle is %s" % fmeta['BeamParticle1'])
info.append(" - Beam 2 particle is %s" % fmeta['BeamParticle2'])
del fmeta['BeamParticle1']
del fmeta['BeamParticle2']
if 'PolarizationB1' in fmeta:
info.append(' - Has %s polarization for Beam 1 and %s for beam 2' %
(fmeta['PolarizationB1'], fmeta['PolarizationB2']))
del fmeta['PolarizationB1']
del fmeta["PolarizationB2"]
if 'Datatype' in fmeta:
if fmeta['Datatype'] == 'gen':
info.append(' - This is a generator level sample')
elif fmeta["Datatype"] == 'SIM':
info.append(" - This is a simulated sample")
elif fmeta['Datatype'] in ['REC', 'DST']:
info.append(' - This is a reconstructed sample')
else:
info.append(' - The datatype is unknown: %s' % fmeta['Datatype'])
del fmeta['Datatype']
if "SWPackages" in fmeta:
info.append(" - Was produced with %s" % ", ".join(fmeta["SWPackages"].split(';')))
del fmeta["SWPackages"]
if "SoftwareTag" in fmeta:
info.append(' - Was produced with %s' % fmeta['SoftwareTag'])
del fmeta['SoftwareTag']
if 'ILDConfig' in fmeta:
info.append(' - Used the %s ILDConfig package' % fmeta["ILDConfig"])
del fmeta["ILDConfig"]
if 'DetectorModel' in fmeta:
info.append(" - Using the %s detector model" % fmeta['DetectorModel'])
del fmeta['DetectorModel']
if 'NumberOfEvents' in fmeta:
info.append(' - Has %s events or less per file' % fmeta['NumberOfEvents'])
del fmeta['NumberOfEvents']
if "CrossSection" in fmeta:
xsec = str(fmeta["CrossSection"])
del fmeta["CrossSection"]
if 'CrossSectionError' in fmeta:
xsec += " +/- "+str(fmeta["CrossSectionError"])
del fmeta["CrossSectionError"]
xsec += " fb"
info.append(" - Cross section %s" % xsec)
if "AdditionalInfo" in fmeta:
try:
dinfo = DEncode.decode(fmeta["AdditionalInfo"])
except:
dinfo = eval(fmeta["AdditionalInfo"])
info.append(" - There is some additional info:")
if type(dinfo) == types.TupleType:
dinfo = dinfo[0]
if type(dinfo) == types.DictType:
dictinfo = dinfo
if 'xsection' in dictinfo:
if 'sum' in dictinfo['xsection']:
if 'xsection' in dictinfo['xsection']['sum']:
xsec= str(dictinfo['xsection']['sum']['xsection'])
if 'err_xsection' in dictinfo['xsection']['sum']:
xsec += ' +/- %s' % dictinfo['xsection']['sum']['err_xsection']
xsec += "fb"
info.append(' Cross section %s' % xsec)
else:
info.append(' %s' % dinfo)
del fmeta["AdditionalInfo"]
if 'Luminosity' in fmeta:
info.append(' - Sample corresponds to a luminosity of %sfb'%fmeta["Luminosity"])
del fmeta['Luminosity']
if 'Ancestors' in fmeta:
if len(fmeta["Ancestors"]):
info.append(" - Was produced from:")
for anc in fmeta["Ancestors"]:
info.append(' %s' % anc)
del fmeta["Ancestors"]
if 'Descendants' in fmeta:
if len(fmeta["Descendants"]):
info.append(" - Gave the following files:")
for des in fmeta["Descendants"]:
info.append(' %s' % des)
del fmeta["Descendants"]
if 'DetectorType' in fmeta:
#We don't need this here
del fmeta['DetectorType']
if fmeta:
info.append('Remaining metadata: %s' % str(fmeta))
return info
def deserializeTask(self, taskStub): """ Tasks are received as a stream of bytes. They have to be converted from that into a usable object. """ return S_OK(DEncode.decode(taskStub)[0])
def execute_request( self ):
""" Takes one DISET request and forward it to the destination service
"""
gMonitor.addMark( "Iteration", 1 )
if self.RequestDB:
res = self.RequestDB.getRequest( 'diset' )
else:
res = self.RequestDBClient.getRequest( 'diset' )
if not res['OK']:
gLogger.error( "DISETForwardingAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "DISETForwardingAgent.execute: No requests to be executed found." )
return S_OK()
gMonitor.addMark( "Attempted", 1 )
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
try:
jobID = int( res['Value']['JobID'] )
except:
jobID = 0
gLogger.info( "DISETForwardingAgent.execute: Obtained request %s" % requestName )
if self.RequestDB:
result = self.RequestDB._getRequestAttribute( 'RequestID', requestName = requestName )
if not result['OK']:
return S_OK( 'Can not get the request execution order' )
requestID = result['Value']
result = self.RequestDB.getCurrentExecutionOrder( requestID )
else:
result = self.RequestDBClient.getCurrentExecutionOrder( requestName )
if result['OK']:
currentOrder = result['Value']
else:
return S_OK( 'Can not get the request execution order' )
oRequest = RequestContainer( request = requestString )
requestAttributes = oRequest.getRequestAttributes()['Value']
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'diset' )
if not res['OK']:
errStr = "DISETForwardingAgent.execute: Failed to obtain number of diset subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "DISETForwardingAgent.execute: Found %s sub requests for job %s" % ( res['Value'], jobID ) )
################################################
# For all the sub-requests in the request
modified = False
for ind in range( res['Value'] ):
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'diset' )['Value']
subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
subStatus = subRequestAttributes['Status']
gLogger.info( "DISETForwardingAgent.execute: Processing sub-request %s with execution order %d" % ( ind, subExecutionOrder ) )
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
operation = subRequestAttributes['Operation']
gLogger.info( "DISETForwardingAgent.execute: Attempting to forward %s type." % operation )
rpcStubString = subRequestAttributes['Arguments']
rpcStub, length = DEncode.decode( rpcStubString )
res = executeRPCStub( rpcStub )
if res['OK']:
gLogger.info( "DISETForwardingAgent.execute: Successfully forwarded." )
oRequest.setSubRequestStatus( ind, 'diset', 'Done' )
gMonitor.addMark( "Successful", 1 )
modified = True
elif res['Message'] == 'No Matching Job':
gLogger.warn( "DISETForwardingAgent.execute: No corresponding job found. Setting to done." )
oRequest.setSubRequestStatus( ind, 'diset', 'Done' )
else:
gLogger.error( "DISETForwardingAgent.execute: Failed to forward request.", res['Message'] )
else:
gLogger.info( "DISETForwardingAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
if self.RequestDB:
res = self.RequestDB.updateRequest( requestName, requestString )
else:
res = self.RequestDBClient.updateRequest( requestName, requestString )
if res['OK']:
gLogger.info( "DISETForwardingAgent.execute: Successfully updated request." )
else:
gLogger.error( "DISETForwardingAgent.execute: Failed to update request" )
if modified and jobID:
result = self.RequestDBClient.finalizeRequest( requestName, jobID )
return S_OK()
def updateAlarm( self, updateReq ):
#Discover alarm identification
idOK = False
for field in self.__updateAlarmIdentificationFields:
if field in updateReq:
idOK = True
if not idOK:
return S_ERROR( "Need at least one field to identify which alarm to update! %s" % self.__updateAlarmIdentificationFields )
if 'alarmKey' in updateReq:
alarmKey = updateReq[ 'alarmKey' ]
result = self.__getAlarmIdFromKey( alarmKey )
if not result[ 'OK' ]:
self.log.error( "Could not get alarm id for key", " %s: %s" % ( alarmKey, result[ 'Value' ] ) )
return result
updateReq[ 'id' ] = result[ 'Value' ]
self.log.info( "Retrieving alarm key %s maps to id %s" % ( alarmKey, updateReq[ 'id' ] ) )
#Check fields
for field in self.__updateAlarmMandatoryFields:
if field not in updateReq:
return S_ERROR( "Oops. Missing %s" % field )
validReq = False
for field in self.__updateAlarmAtLeastOneField:
if field in updateReq:
validReq = True
if not validReq:
return S_OK( "Requirement needs at least one of %s" % " ".join( self.__updateAlarmAtLeastOneField ) )
author = updateReq[ 'author' ]
followers = [ author ]
if author not in CS.getAllUsers():
return S_ERROR( "%s is not a known user" % author )
result = self._escapeString( author )
if not result[ 'OK' ]:
return result
author = result[ 'Value' ]
try:
alarmId = int( updateReq[ 'id' ] )
except:
return S_ERROR( "Oops, Alarm id is not valid! (bad boy...)" )
result = self._query( "SELECT AlarmId FROM `ntf_Alarms` WHERE AlarmId=%d" % alarmId )
if not result[ 'OK' ]:
return result
if not result[ 'Value' ]:
return S_ERROR( "Alarm %s does not exist!" % alarmId )
sqlFields = [ 'AlarmId', 'Author', 'Timestamp' ]
sqlValues = [ "%d" % alarmId, author, 'UTC_TIMESTAMP()' ]
rawComment = ""
if 'comment' in updateReq:
rawComment = updateReq[ 'comment' ]
result = self._escapeString( rawComment )
if not result[ 'OK' ]:
return result
sqlFields.append( "Comment" )
sqlValues.append( result[ 'Value' ] )
modifications = False
if 'modifications' in updateReq:
modifications = updateReq[ 'modifications' ]
result = self.__processUpdateAlarmModifications( modifications )
if not result[ 'OK' ]:
return result
alarmModsSQL, encodedMods, newFollowers = result[ 'Value' ]
sqlFields.append( "Modifications" )
result = self._escapeString( encodedMods )
if not result[ 'OK' ]:
return result
sqlValues.append( result[ 'Value' ] )
if newFollowers:
followers.extend( newFollowers )
logSQL = "INSERT INTO `ntf_AlarmLog` (%s) VALUES (%s)" % ( ",".join( sqlFields ), ",".join( sqlValues ) )
result = self._update( logSQL )
if not result[ 'OK' ]:
return result
modSQL = "ModTime=UTC_TIMESTAMP()"
if modifications:
modSQL = "%s, %s" % ( modSQL, alarmModsSQL )
updateSQL = "UPDATE `ntf_Alarms` SET %s WHERE AlarmId=%d" % ( modSQL, alarmId )
result = self._update( updateSQL )
if not result[ 'OK' ]:
return result
#Get notifications config
sqlQuery = "SELECT Notifications FROM `ntf_Alarms` WHERE AlarmId=%s" % alarmId
result = self._query( sqlQuery )
if not result[ 'OK' ] or not result[ 'Value' ]:
self.log.error( "Could not retrieve default notifications for alarm", "%s" % alarmId )
return S_OK( alarmId )
notificationsDict = DEncode.decode( result[ 'Value' ][0][0] )[0]
for v in self.__validAlarmNotifications:
if v not in notificationsDict:
notificationsDict[ v ] = 0
for follower in followers:
result = self.modifyFollowerForAlarm( alarmId, follower, notificationsDict, overwrite = False )
if not result[ 'OK' ]:
varMsg = "\nFollower: %s\nAlarm: %s\nError: %s" % ( follower, alarmId, result['Message'] )
self.log.error( "Couldn't set follower for alarm", varMsg )
return self.__notifyAlarm( alarmId )
if compressType == 'Z':
gLogger.info( "Compressed request, uncompressing" )
try:
stub = base64.urlsafe_b64decode( stub )
except Exception, e:
gLogger.error( "Oops! Plot request is not properly encoded!", str( e ) )
return S_ERROR( "Oops! Plot request is not properly encoded!: %s" % str( e ) )
try:
stub = zlib.decompress( stub )
except Exception, e:
gLogger.error( "Oops! Plot request is invalid!", str( e ) )
return S_ERROR( "Oops! Plot request is invalid!: %s" % str( e ) )
elif compressType == 'S':
gLogger.info( "Base64 request, decoding" )
try:
stub = base64.urlsafe_b64decode( stub )
except Exception, e:
gLogger.error( "Oops! Plot request is not properly encoded!", str( e ) )
return S_ERROR( "Oops! Plot request is not properly encoded!: %s" % str( e ) )
elif compressType == 'R':
#Do nothing, it's already uncompressed
pass
else:
gLogger.error( "Oops! Stub type '%s' is unknown :P" % compressType )
return S_ERROR( "Oops! Stub type '%s' is unknown :P" % compressType )
plotRequest, stubLength = DEncode.decode( stub )
if len( stub ) != stubLength:
gLogger.error( "Oops! The stub is longer than the data :P" )
return S_ERROR( "Oops! The stub is longer than the data :P" )
return S_OK( plotRequest )
def __decodeVar( self, data, dataTypeRE ):
try:
dataObj, lenData = DEncode.decode( data )
except Exception, e:
return S_ERROR( "Cannot decode data: %s" % str( e ) )
