def web_listPublicDesktopStates( self ):
up = self.__getUP()
result = yield self.threadTask( up.listAvailableVars )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
data = result['Value']
paramNames = ['UserName', 'Group', 'VO', 'desktop']
records = []
for i in data:
records += [dict( zip( paramNames, i ) )]
sharedDesktops = {}
for i in records:
result = yield self.threadTask( up.getVarPermissions, i['desktop'] )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
if result['Value']['ReadAccess'] == 'ALL':
print i['UserName'], i['Group'], i
result = yield self.threadTask( up.retrieveVarFromUser , i['UserName'], i['Group'], i['desktop'] )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
if i['UserName'] not in sharedDesktops:
sharedDesktops[i['UserName']] = {}
sharedDesktops[i['UserName']][i['desktop']] = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( result['Value'] ) ) )[0] )
sharedDesktops[i['UserName']]['Metadata'] = i
else:
sharedDesktops[i['UserName']][i['desktop']] = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( result['Value'] ) ) )[0] )
sharedDesktops[i['UserName']]['Metadata'] = i
self.finish( sharedDesktops )
def __executePythonFunction(self, function, writePipe, *stArgs,
**stKeyArgs):
"""
execute function :funtion: using :stArgs: and :stKeyArgs:
"""
from DIRAC.Core.Utilities import DEncode
try:
os.write(writePipe,
DEncode.encode(S_OK(function(*stArgs, **stKeyArgs))))
except OSError as x:
if str(x) == "[Errno 32] Broken pipe":
# the parent has died
pass
except Exception as x:
self.log.exception("Exception while executing", function.__name__)
os.write(writePipe, DEncode.encode(S_ERROR(str(x))))
# HACK: Allow some time to flush logs
time.sleep(1)
try:
os.close(writePipe)
finally:
os._exit(0)
def __listToXML(self, name, aList, indent=0, attributes={}):
""" Utility to convert a list to XML
"""
"""
xml_attributes = ''
xml_elements = []
for element in list:
if type(element) is DictType:
xml_elements.append(self.__dictionaryToXML(name[:-1],element,indent+1))
elif type(value) is ListType:
xml_elements.append(self.__listToXML(name[:-1],element,indent+1))
else:
xml_attributes += ' '*(indent+1)*8+'<%s element_type="leaf"><![CDATA[%s]]></%s>\n' % (name[:-1],str(element),name[:-1])
for attr,value in attributes.items():
xml_attributes += ' '*(indent+1)*8+'<%s element_type="leaf"><![CDATA[%s]]></%s>\n' % (attr,str(value),attr)
out = ' '*indent*8+'<%s element_type="list">\n%s\n' % (name,xml_attributes[:-1])
for el in xml_elements:
out += ' '*indent*8+el
out += ' '*indent*8+'</%s>\n' % name
"""
out = ''
if aList:
den = DEncode.encode(aList)
out += ' ' * indent * 8 + '<%s element_type="list">\n' % (name)
out += ' ' * (
indent + 1
) * 8 + '<EncodedString element_type="leaf"><![CDATA[%s]]></EncodedString>\n' % (
den)
out += ' ' * indent * 8 + '</%s>\n' % name
return out
def serialize( self ):
if self.__manifest:
manifest = ( self.__manifest.dumpAsCFG(), self.__manifest.isDirty() )
else:
manifest = None
return DEncode.encode( ( self.__jid, self.__cache, self.__jobLog, manifest,
self.__initState, self.__insertIntoTQ, tuple( self.__dirtyKeys ) ) )
def getAlarmLog(self, alarmId):
try:
alarmId = int(alarmId)
except Exception:
return S_ERROR("Alarm id must be a non decimal number")
sqlSel = "SELECT %s FROM `ntf_AlarmLog` WHERE AlarmId=%d ORDER BY Timestamp ASC" % (
",".join(self.__alarmLogFields),
alarmId,
)
result = self._query(sqlSel)
if not result["OK"]:
return result
decodedRows = []
for row in result["Value"]:
decodedRows.append(list(row))
if not row[3]:
decodedRows.append(list(row))
continue
dec = DEncode.decode(row[3])
decodedRows[-1][3] = dec[0]
resultDict = {}
resultDict["ParameterNames"] = self.__alarmLogFields
resultDict["Records"] = decodedRows
return S_OK(resultDict)
def extractRequestFromFileId( fileId ):
stub = fileId[2:]
compressType = fileId[0]
if compressType == 'Z':
gLogger.info( "Compressed request, uncompressing" )
try:
stub = base64.urlsafe_b64decode( stub )
except Exception as e:
gLogger.error( "Oops! Plot request is not properly encoded!", str( e ) )
return S_ERROR( "Oops! Plot request is not properly encoded!: %s" % str( e ) )
try:
stub = zlib.decompress( stub )
except Exception as e:
gLogger.error( "Oops! Plot request is invalid!", str( e ) )
return S_ERROR( "Oops! Plot request is invalid!: %s" % str( e ) )
elif compressType == 'S':
gLogger.info( "Base64 request, decoding" )
try:
stub = base64.urlsafe_b64decode( stub )
except Exception as e:
gLogger.error( "Oops! Plot request is not properly encoded!", str( e ) )
return S_ERROR( "Oops! Plot request is not properly encoded!: %s" % str( e ) )
elif compressType == 'R':
#Do nothing, it's already uncompressed
pass
else:
gLogger.error( "Oops! Stub type is unknown", compressType )
return S_ERROR( "Oops! Stub type '%s' is unknown :P" % compressType )
plotRequest, stubLength = DEncode.decode( stub )
if len( stub ) != stubLength:
gLogger.error( "Oops! The stub is longer than the data :P" )
return S_ERROR( "Oops! The stub is longer than the data :P" )
return S_OK( plotRequest )
def applicationSpecificInputs(self):
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
if self.workflow_commons.has_key('ProductionOutputData'):
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
self.nbofevents = self.NumberOfEvents
if self.workflow_commons.has_key('Luminosity'):
self.luminosity = self.workflow_commons['Luminosity']
##Additional info: cross section only for the time being, comes from WHIZARD
if self.workflow_commons.has_key('Info'):
if 'stdhepcut' in self.workflow_commons['Info']:
self.sel_eff = self.workflow_commons['Info']['stdhepcut']['Reduction']
self.cut_eff = self.workflow_commons['Info']['stdhepcut']['CutEfficiency']
del self.workflow_commons['Info']['stdhepcut']
self.add_info = DEncode.encode(self.workflow_commons['Info'])
return S_OK('Parameters resolved')
def web_saveAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
state = self.request.arguments['state'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
data = base64.b64encode(zlib.compress(DEncode.encode(state), 9))
# before we save the state (modify the state) we have to remeber the actual access: ReadAccess and PublishAccess
result = yield self.threadTask(up.getVarPermissions, name)
if result['OK']:
access = result['Value']
else:
access = {
'ReadAccess': 'USER',
'PublishAccess': 'USER'
} # this is when the application/desktop does not exists.
result = yield self.threadTask(up.storeVar, name, data)
if not result['OK']:
raise WErr.fromSERROR(result)
# change the access to the application/desktop
result = yield self.threadTask(up.setVarPermissions, name, access)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
def __executePythonFunction(self, function, writePipe, *stArgs, **stKeyArgs):
try:
os.write(writePipe, DEncode.encode(S_OK(function(*stArgs, **stKeyArgs))))
except OSError, x:
if str(x) == "[Errno 32] Broken pipe":
# the parent has died
pass
def __listToXML( self, name, aList, indent = 0, attributes = {} ):
""" Utility to convert a list to XML
"""
"""
xml_attributes = ''
xml_elements = []
for element in list:
if type(element) is DictType:
xml_elements.append(self.__dictionaryToXML(name[:-1],element,indent+1))
elif type(value) is ListType:
xml_elements.append(self.__listToXML(name[:-1],element,indent+1))
else:
xml_attributes += ' '*(indent+1)*8+'<%s element_type="leaf"><![CDATA[%s]]></%s>\n' % (name[:-1],str(element),name[:-1])
for attr,value in attributes.items():
xml_attributes += ' '*(indent+1)*8+'<%s element_type="leaf"><![CDATA[%s]]></%s>\n' % (attr,str(value),attr)
out = ' '*indent*8+'<%s element_type="list">\n%s\n' % (name,xml_attributes[:-1])
for el in xml_elements:
out += ' '*indent*8+el
out += ' '*indent*8+'</%s>\n' % name
"""
out = ''
if aList:
den = DEncode.encode( aList )
out += ' ' * indent * 8 + '<%s element_type="list">\n' % ( name )
out += ' ' * ( indent + 1 ) * 8 + '<EncodedString element_type="leaf"><![CDATA[%s]]></EncodedString>\n' % ( den )
out += ' ' * indent * 8 + '</%s>\n' % name
return out
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = "MyRequest"
op = Operation()
switches = {}
archiveLFN = "/vo/tars/myTar.tar"
op.Arguments = DEncode.encode({
"SourceSE":
switches.get("SourceSE", "SOURCE-SE"),
"TarballSE":
switches.get("TarballSE", "TARBALL-SE"),
"RegisterDescendent":
False,
"ArchiveLFN":
archiveLFN,
})
op.Type = "ArchiveFiles"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def getFileInfo(lfn):
""" Retrieve the file info
"""
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.Core.Utilities import DEncode
from DIRAC import gLogger
fc = FileCatalogClient()
lumi = 0
nbevts = 0
res = fc.getFileUserMetadata(lfn)
if not res['OK']:
gLogger.error("Failed to get metadata of %s" % lfn)
return (0,0,{})
if res['Value'].has_key('Luminosity'):
lumi += float(res['Value']['Luminosity'])
addinfo = {}
if 'AdditionalInfo' in res['Value']:
addinfo = res['Value']['AdditionalInfo']
if addinfo.count("{"):
addinfo = eval(addinfo)
else:
addinfo = DEncode.decode(addinfo)[0]
if "NumberOfEvents" in res['Value'].keys():
nbevts += int(res['Value']['NumberOfEvents'])
return (float(lumi),int(nbevts),addinfo)
def serialize(self):
if self.__manifest:
manifest = (self.__manifest.dumpAsCFG(), self.__manifest.isDirty())
else:
manifest = None
return DEncode.encode((self.__jid, self.__cache, self.__jobLog, manifest,
self.__initState, self.__insertIntoTQ, tuple(self.__dirtyKeys)))
def printOperation( indexOperation, verbose = True, onlyFailed = False ):
global output
i, op = indexOperation
prStr = ''
if op.SourceSE:
prStr += 'SourceSE: %s' % op.SourceSE
if op.TargetSE:
prStr += ( ' - ' if prStr else '' ) + 'TargetSE: %s' % op.TargetSE
if prStr:
prStr += ' - '
prStr += 'Created %s, Updated %s' % ( op.CreationTime, op.LastUpdate )
if op.Type == 'ForwardDISET':
from DIRAC.Core.Utilities import DEncode
decode, _length = DEncode.decode( op.Arguments )
if verbose:
output = ''
prettyPrint( decode, offset = 10 )
prStr += '\n Arguments:\n' + output.strip( '\n' )
else:
prStr += '\n Service: %s' % decode[0][0]
gLogger.always( " [%s] Operation Type='%s' ID=%s Order=%s Status='%s'%s%s" % ( i, op.Type, op.OperationID,
op.Order, op.Status,
( " Error='%s'" % op.Error ) if op.Error and op.Error.strip() else "",
( " Catalog=%s" % op.Catalog ) if op.Catalog else "" ) )
if prStr:
gLogger.always( " %s" % prStr )
for indexFile in enumerate( op ):
if not onlyFailed or indexFile[1].Status == 'Failed':
printFile( indexFile )
def _reportFileUsage(self, dirDict):
"""Send the data usage report (site,dirDict) where dirDict = {'Dataset':NumberOfHits}
example: {'/lhcb/certification/test/ALLSTREAMS.DST/00000002/0000/': 1,
'/lhcb/LHCb/Collision11/BHADRON.DST/00012957/0000/': 2}
"""
self.log.verbose('Reporting input file usage:')
for entry in dirDict:
self.log.verbose('%s:%s' % (entry, dirDict[entry]))
# dataUsageClient = RPCClient( 'DataManagement/DataUsage', timeout = 120 )
localSite = gConfig.getValue('/LocalSite/Site', 'UNKNOWN')
try:
localSite = localSite.split('.')[1]
except BaseException:
pass
self.log.verbose('Using Site Name: %s' % (localSite))
if self._enableModule():
usageStatus = self.dataUsageClient.sendDataUsageReport(localSite, dirDict)
if not usageStatus['OK']:
self.log.error('Could not send data usage report, preparing a DISET failover request object')
self.log.verbose(usageStatus['rpcStub'])
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode(usageStatus['rpcStub'])
self.request.addOperation(forwardDISETOp)
self.workflow_commons['Request'] = self.request
else:
self.log.info('Would have attempted to report %s at %s' % (dirDict, localSite))
return S_OK()
return S_OK()
def __executePythonFunction( self, function, writePipe, *stArgs, **stKeyArgs ):
try:
os.write( writePipe, DEncode.encode( S_OK( function( *stArgs, **stKeyArgs ) ) ) )
except OSError, x:
if str( x ) == '[Errno 32] Broken pipe':
# the parent has died
pass
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
switches = {}
archiveLFN = '/vo/tars/myTar.tar'
op.Arguments = DEncode.encode({
'SourceSE':
switches.get('SourceSE', 'SOURCE-SE'),
'TarballSE':
switches.get('TarballSE', 'TARBALL-SE'),
'RegisterDescendent':
False,
'ArchiveLFN':
archiveLFN
})
op.Type = 'ArchiveFiles'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
def receiveData( self, maxBufferSize = 0, blockAfterKeepAlive = True, idleReceive = False ):
self.__updateLastActionTimestamp()
if self.receivedMessages:
return self.receivedMessages.pop( 0 )
#Buffer size can't be less than 0
maxBufferSize = max( maxBufferSize, 0 )
try:
#Look either for message length of keep alive magic string
iSeparatorPosition = self.byteStream.find( ":", 0, 10 )
keepAliveMagicLen = len( BaseTransport.keepAliveMagic )
isKeepAlive = self.byteStream.find( BaseTransport.keepAliveMagic, 0, keepAliveMagicLen ) == 0
#While not found the message length or the ka, keep receiving
while iSeparatorPosition == -1 and not isKeepAlive:
retVal = self._read( 1024 )
#If error return
if not retVal[ 'OK' ]:
return retVal
#If closed return error
if not retVal[ 'Value' ]:
return S_ERROR( "Peer closed connection" )
#New data!
self.byteStream += retVal[ 'Value' ]
#Look again for either message length of ka magic string
iSeparatorPosition = self.byteStream.find( ":", 0, 10 )
isKeepAlive = self.byteStream.find( BaseTransport.keepAliveMagic, 0, keepAliveMagicLen ) == 0
#Over the limit?
if maxBufferSize and len( self.byteStream ) > maxBufferSize and iSeparatorPosition == -1 :
return S_ERROR( "Read limit exceeded (%s chars)" % maxBufferSize )
#Keep alive magic!
if isKeepAlive:
gLogger.debug( "Received keep alive header" )
#Remove the ka magic from the buffer and process the keep alive
self.byteStream = self.byteStream[ keepAliveMagicLen: ]
return self.__processKeepAlive( maxBufferSize, blockAfterKeepAlive )
#From here it must be a real message!
#Process the size and remove the msg length from the bytestream
size = int( self.byteStream[ :iSeparatorPosition ] )
self.byteStream = self.byteStream[ iSeparatorPosition + 1: ]
#Receive while there's still data to be received
while len( self.byteStream ) < size:
retVal = self._read( size - len( self.byteStream ), skipReadyCheck = True )
if not retVal[ 'OK' ]:
return retVal
if not retVal[ 'Value' ]:
return S_ERROR( "Peer closed connection" )
self.byteStream += retVal[ 'Value' ]
if maxBufferSize and len( self.byteStream ) > maxBufferSize:
return S_ERROR( "Read limit exceeded (%s chars)" % maxBufferSize )
#Data is here! take it out from the bytestream, dencode and return
data = self.byteStream[ :size ]
self.byteStream = self.byteStream[ size: ]
try:
data = DEncode.decode( data )[0]
except Exception, e:
return S_ERROR( "Could not decode received data: %s" % str( e ) )
if idleReceive:
self.receivedMessages.append( data )
return S_OK()
return data
def receiveData( self, maxBufferSize = 0, blockAfterKeepAlive = True, idleReceive = False ):
self.__updateLastActionTimestamp()
if self.receivedMessages:
return self.receivedMessages.pop( 0 )
#Buffer size can't be less than 0
maxBufferSize = max( maxBufferSize, 0 )
try:
#Look either for message length of keep alive magic string
iSeparatorPosition = self.byteStream.find( ":", 0, 10 )
keepAliveMagicLen = len( BaseTransport.keepAliveMagic )
isKeepAlive = self.byteStream.find( BaseTransport.keepAliveMagic, 0, keepAliveMagicLen ) == 0
#While not found the message length or the ka, keep receiving
while iSeparatorPosition == -1 and not isKeepAlive:
retVal = self._read( 1024 )
#If error return
if not retVal[ 'OK' ]:
return retVal
#If closed return error
if not retVal[ 'Value' ]:
return S_ERROR( "Peer closed connection" )
#New data!
self.byteStream += retVal[ 'Value' ]
#Look again for either message length of ka magic string
iSeparatorPosition = self.byteStream.find( ":", 0, 10 )
isKeepAlive = self.byteStream.find( BaseTransport.keepAliveMagic, 0, keepAliveMagicLen ) == 0
#Over the limit?
if maxBufferSize and len( self.byteStream ) > maxBufferSize and iSeparatorPosition == -1 :
return S_ERROR( "Read limit exceeded (%s chars)" % maxBufferSize )
#Keep alive magic!
if isKeepAlive:
gLogger.debug( "Received keep alive header" )
#Remove the ka magic from the buffer and process the keep alive
self.byteStream = self.byteStream[ keepAliveMagicLen: ]
return self.__processKeepAlive( maxBufferSize, blockAfterKeepAlive )
#From here it must be a real message!
#Process the size and remove the msg length from the bytestream
size = int( self.byteStream[ :iSeparatorPosition ] )
self.byteStream = self.byteStream[ iSeparatorPosition + 1: ]
#Receive while there's still data to be received
while len( self.byteStream ) < size:
retVal = self._read( size - len( self.byteStream ), skipReadyCheck = True )
if not retVal[ 'OK' ]:
return retVal
if not retVal[ 'Value' ]:
return S_ERROR( "Peer closed connection" )
self.byteStream += retVal[ 'Value' ]
if maxBufferSize and len( self.byteStream ) > maxBufferSize:
return S_ERROR( "Read limit exceeded (%s chars)" % maxBufferSize )
#Data is here! take it out from the bytestream, dencode and return
data = self.byteStream[ :size ]
self.byteStream = self.byteStream[ size: ]
try:
data = DEncode.decode( data )[0]
except Exception, e:
return S_ERROR( "Could not decode received data: %s" % str( e ) )
if idleReceive:
self.receivedMessages.append( data )
return S_OK()
return data
def encode(inData):
"""Encode the input data
:param inData: data to be encoded
:return: an encoded string
"""
if os.getenv("DIRAC_USE_JSON_ENCODE", "NO").lower() in ("yes", "true"):
return JEncode.encode(inData)
return DEncode.encode(inData)
def __decodeVar( self, data, dataTypeRE ):
try:
dataObj, lenData = DEncode.decode( data )
except Exception as e:
return S_ERROR( "Cannot decode data: %s" % str( e ) )
if dataTypeRE:
result = self.checkTypeRe( dataObj, dataTypeRE )
if not result[ 'OK' ]:
return result
return S_OK( dataObj )
def __listToXML(self,name,list,indent = 0):
""" Utility to convert a list to XML
"""
out = ''
if list:
den = DEncode.encode(list)
out += ' '*indent*8+'<%s element_type="list">\n' % (name)
out += ' '*(indent+1)*8+'<EncodedString element_type="leaf"><![CDATA[%s]]></EncodedString>\n' % (den)
out += ' '*indent*8+'</%s>\n' % name
return out
def web_listAppState( self ):
up = self.__getUP()
result = yield self.threadTask( up.retrieveAllVars )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
data = result[ 'Value' ]
for k in data:
#Unpack data
data[ k ] = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( data[ k ] ) ) )[0] )
self.finish( data )
def export_tryView( self, fromSecs, toSecs, viewDescriptionStub ):
"""
Generates plots based on a DEncoded view description
"""
viewDescription, _stubLength = DEncode.decode( viewDescriptionStub )
if not 'definition' in viewDescription:
return S_ERROR( "No plot definition given" )
defDict = viewDescription[ 'definition' ]
defDict[ 'sources.setup' ] = self.serviceInfoDict[ 'clientSetup' ]
return gServiceInterface.generatePlots( fromSecs, toSecs, viewDescription )
def encode(inData):
""" Encode the input data
:param inData: data to be encoded
:return: an encoded string
"""
if os.getenv('DIRAC_USE_JSON_ENCODE', 'NO').lower() in ('yes', 'true'):
return JEncode.encode(inData)
return DEncode.encode(inData)
def decode(encodedData):
""" Decode the encoded string
:param encodedData: encoded string
:return: the decoded objects, encoded object length
"""
if os.getenv('DIRAC_USE_JSON_DECODE', 'NO').lower() in ('yes', 'true'):
try:
# 'null' is a special case.
# None is encoded as 'null' as JSON
# that DEncode would understand as None, since it starts with 'n'
# but the length of the decoded string will not be correct
if encodedData == 'null':
raise Exception
return DEncode.decode(encodedData)
except Exception:
return JEncode.decode(encodedData)
return DEncode.decode(encodedData)
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def setUp( self ):
""" test set up """
self.hiArgs = ( ( "RequestManagement/RequestManager",
{ "keepAliveLapse": 10, "timeout": 5 } ),
"foo",
( 12345, { "Hi": "There!" } ) )
self.req = Request( { "RequestName": "testRequest" } )
self.op = Operation( { "Type": "ForwardDISET",
"Arguments": DEncode.encode( self.hiArgs ) } )
self.req += self.op
def __executePythonFunction( self, function, writePipe, *stArgs, **stKeyArgs ):
"""
execute function :funtion: using :stArgs: and :stKeyArgs:
"""
try:
os.write( writePipe, DEncode.encode( S_OK( function( *stArgs, **stKeyArgs ) ) ) )
except OSError, x:
if str( x ) == '[Errno 32] Broken pipe':
# the parent has died
pass
def __call__( self ):
""" It expects to find the arguments for tc.setFileStatusForTransformation in operation.Arguments
"""
try:
setFileStatusDict = DEncode.decode( self.operation.Arguments )[0]
self.log.debug( "decoded filStatusDict=%s" % str( setFileStatusDict ) )
except ValueError, error:
self.log.exception( error )
self.operation.Error = str( error )
self.operation.Status = "Failed"
return S_ERROR( str( error ) )
def __call__( self ):
""" execute RPC stub """
# # decode arguments
try:
decode, length = DEncode.decode( self.operation.Arguments )
self.log.debug( "decoded len=%s val=%s" % ( length, decode ) )
except ValueError, error:
self.log.exception( error )
self.operation.Error = str( error )
self.operation.Status = "Failed"
return S_ERROR( str( error ) )
def web_changeView( self ):
up = self.__getUP()
try:
desktopName = self.request.arguments[ 'desktop' ][-1]
view = self.request.arguments[ 'view' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
result = yield self.threadTask( up.retrieveVar, desktopName )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
data = result['Value']
oDesktop = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( data ) ) )[0] )
oDesktop[unicode( 'view' )] = unicode( view )
oDesktop = json.dumps( oDesktop )
data = base64.b64encode( zlib.compress( DEncode.encode( oDesktop ), 9 ) )
result = yield self.threadTask( up.storeVar, desktopName, data )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
self.set_status( 200 )
self.finish()
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
commitOp = None
if not result['OK']:
# Generate Request
commitOp = Operation()
commitOp.Type = 'SetFileStatus'
commitOp.Arguments = DEncode.encode( {'transformation':self.transformation, 'statusDict':self.statusDict, 'force':self.force} )
return S_OK( commitOp )
def export_saveView( self, viewName, viewDescriptionStub ):
"""
Saves a view
"""
if len( viewName ) == 0:
return S_OK( "View name not valid" )
viewDescription, _stubLength = DEncode.decode( viewDescriptionStub )
if not 'definition' in viewDescription:
return S_ERROR( "No plot definition given" )
defDict = viewDescription[ 'definition' ]
defDict[ 'sources.setup' ] = self.serviceInfoDict[ 'clientSetup' ]
return gServiceInterface.saveView( viewName, viewDescription )
def web_loadAppState( self ):
up = self.__getUP()
try:
name = self.request.arguments[ 'name' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
result = yield self.threadTask( up.retrieveVar, name )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result)
data = result[ 'Value' ]
data, count = DEncode.decode( zlib.decompress( base64.b64decode( data ) ) )
self.finish( data )
def __listToXML_new( self, xmldoc, tagName, aList ): # # create tag = xmldoc.createElement( tagName ) tag.setAttribute( "element_type", "list" ) encodedStringTag = xmldoc.createElement( "EncodedString" ) encodedStringTag.setAttribute( "element_type", "leaf" ) encodedStringCDATA = xmldoc.createCDATASection( DEncode.encode( aList ) ) # # appending encodedStringTag.appendChild( encodedStringCDATA ) tag.appendChild( encodedStringTag ) # # returning return tag
def web_saveAppState( self ):
up = self.__getUP()
try:
name = self.request.arguments[ 'name' ][-1]
state = self.request.arguments[ 'state' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
data = base64.b64encode( zlib.compress( DEncode.encode( state ), 9 ) )
result = yield self.threadTask( up.storeVar, name, data )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
self.set_status( 200 )
self.finish()
def retrieveAllVars( self ):
rpcClient = self.__getRPCClient()
result = rpcClient.retrieveProfileAllVars( self.profile )
if not result[ 'OK' ]:
return result
try:
encodedData = result[ 'Value' ]
dataObj = {}
for k in encodedData:
v, lenData = DEncode.decode( encodedData[k] )
dataObj[ k ] = v
except Exception, e:
return S_ERROR( "Cannot decode data: %s" % str( e ) )
def retrieveOptimizerParam(self, name):
if not self.__jobData.jobState:
return S_ERROR("This function can only be called inside the optimizeJob function")
result = self.__jobData.jobState.getOptParameter(name)
if not result['OK']:
return result
valenc = result['Value']
try:
value, encLength = DEncode.decode(valenc)
if encLength == len(valenc):
return S_OK(value)
except Exception:
self.jobLog.warn("Opt param %s doesn't seem to be dencoded %s" % (name, valenc))
return S_OK(eval(valenc))
def __call__( self ):
""" execute RPC stub """
# # decode arguments
try:
decode, length = DEncode.decode( self.operation.Arguments )
# FIXME: to be removed when the problem is fixed in the LHCb HLT farm
if 'CPUFactor_to_be_replace' in self.operation.Arguments and len( decode ) == 3 and type( decode[2] ) == type( tuple() ):
decode = ( decode[0], decode[1], tuple( [x.replace( 'CPUFactor_to_be_replace', '0' ) for x in decode[2]] ) )
self.log.debug( "decoded len=%s val=%s" % ( length, decode ) )
except ValueError, error:
self.log.exception( error )
self.operation.Error = str( error )
self.operation.Status = "Failed"
return S_ERROR( str( error ) )
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if "FailedResults" in result:
for res in result['FailedResults']:
if 'rpcStub' in res:
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( res['rpcStub'] )
return S_OK( forwardDISETOp )
def __executePythonFunction(self, function, writePipe, *stArgs, **stKeyArgs):
"""
execute function :funtion: using :stArgs: and :stKeyArgs:
"""
from DIRAC.Core.Utilities import DEncode
try:
os.write(writePipe, DEncode.encode(S_OK(function(*stArgs, **stKeyArgs))))
except OSError as x:
if str(x) == '[Errno 32] Broken pipe':
# the parent has died
pass
except Exception as x:
self.log.exception('Exception while executing', function.__name__)
os.write(writePipe, DEncode.encode(S_ERROR(str(x))))
# HACK: Allow some time to flush logs
time.sleep(1)
try:
os.close(writePipe)
finally:
os._exit(0)
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if result.has_key( 'rpcStub' ):
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( result['rpcStub'] )
else:
return S_ERROR( 'Could not create ForwardDISET operation' )
return S_OK( forwardDISETOp )
