def __executePythonFunction(self, function, writePipe, *stArgs,
**stKeyArgs):
"""
execute function :funtion: using :stArgs: and :stKeyArgs:
"""
from DIRAC.Core.Utilities import DEncode
try:
os.write(writePipe,
DEncode.encode(S_OK(function(*stArgs, **stKeyArgs))))
except OSError as x:
if str(x) == "[Errno 32] Broken pipe":
# the parent has died
pass
except Exception as x:
self.log.exception("Exception while executing", function.__name__)
os.write(writePipe, DEncode.encode(S_ERROR(str(x))))
# HACK: Allow some time to flush logs
time.sleep(1)
try:
os.close(writePipe)
finally:
os._exit(0)
def storeVar(self, varName, data, perms={}):
try:
stub = DEncode.encode(data)
except Exception as e:
return S_ERROR("Cannot encode data:%s" % str(e))
return self.__getRPCClient().storeProfileVar(self.profile, varName,
stub, perms)
def __listToXML( self, name, aList, indent = 0, attributes = {} ):
""" Utility to convert a list to XML
"""
"""
xml_attributes = ''
xml_elements = []
for element in list:
if type(element) is DictType:
xml_elements.append(self.__dictionaryToXML(name[:-1],element,indent+1))
elif type(value) is ListType:
xml_elements.append(self.__listToXML(name[:-1],element,indent+1))
else:
xml_attributes += ' '*(indent+1)*8+'<%s element_type="leaf"><![CDATA[%s]]></%s>\n' % (name[:-1],str(element),name[:-1])
for attr,value in attributes.items():
xml_attributes += ' '*(indent+1)*8+'<%s element_type="leaf"><![CDATA[%s]]></%s>\n' % (attr,str(value),attr)
out = ' '*indent*8+'<%s element_type="list">\n%s\n' % (name,xml_attributes[:-1])
for el in xml_elements:
out += ' '*indent*8+el
out += ' '*indent*8+'</%s>\n' % name
"""
out = ''
if aList:
den = DEncode.encode( aList )
out += ' ' * indent * 8 + '<%s element_type="list">\n' % ( name )
out += ' ' * ( indent + 1 ) * 8 + '<EncodedString element_type="leaf"><![CDATA[%s]]></EncodedString>\n' % ( den )
out += ' ' * indent * 8 + '</%s>\n' % name
return out
def sendData(self, uData, prefix=False):
self.__updateLastActionTimestamp()
sCodedData = DEncode.encode(uData)
if prefix:
dataToSend = "%s%s:%s" % (prefix, len(sCodedData), sCodedData)
else:
dataToSend = "%s:%s" % (len(sCodedData), sCodedData)
for index in range(0, len(dataToSend), self.packetSize):
bytesToSend = min(self.packetSize, len(dataToSend) - index)
packSentBytes = 0
while packSentBytes < bytesToSend:
try:
result = self._write(
dataToSend[index + packSentBytes:index + bytesToSend])
if not result['OK']:
return result
sentBytes = result['Value']
except Exception as e:
return S_ERROR("Exception while sending data: %s" % e)
if sentBytes == 0:
return S_ERROR("Connection closed by peer")
packSentBytes += sentBytes
del sCodedData
sCodedData = None
return S_OK()
def web_saveAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
state = self.request.arguments['state'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
data = base64.b64encode(zlib.compress(DEncode.encode(state), 9))
# before we save the state (modify the state) we have to remeber the actual access: ReadAccess and PublishAccess
result = yield self.threadTask(up.getVarPermissions, name)
if result['OK']:
access = result['Value']
else:
access = {
'ReadAccess': 'USER',
'PublishAccess': 'USER'
} # this is when the application/desktop does not exists.
result = yield self.threadTask(up.storeVar, name, data)
if not result['OK']:
raise WErr.fromSERROR(result)
# change the access to the application/desktop
result = yield self.threadTask(up.setVarPermissions, name, access)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
def __executePythonFunction(self, function, writePipe, *stArgs, **stKeyArgs):
try:
os.write(writePipe, DEncode.encode(S_OK(function(*stArgs, **stKeyArgs))))
except OSError, x:
if str(x) == "[Errno 32] Broken pipe":
# the parent has died
pass
def storeOptimizerParam(self, name, value):
if not self.__jobData.jobState:
return S_ERROR(
"This function can only be called inside the optimizeJob function"
)
valenc = DEncode.encode(value)
return self.__jobData.jobState.setOptParameter(name, valenc)
def __executePythonFunction( self, function, writePipe, *stArgs, **stKeyArgs ):
try:
os.write( writePipe, DEncode.encode( S_OK( function( *stArgs, **stKeyArgs ) ) ) )
except OSError, x:
if str( x ) == '[Errno 32] Broken pipe':
# the parent has died
pass
def serialize(self):
if self.__manifest:
manifest = (self.__manifest.dumpAsCFG(), self.__manifest.isDirty())
else:
manifest = None
return DEncode.encode((self.__jid, self.__cache, self.__jobLog, manifest,
self.__initState, self.__insertIntoTQ, tuple(self.__dirtyKeys)))
def applicationSpecificInputs(self):
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
if self.workflow_commons.has_key('ProductionOutputData'):
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
self.nbofevents = self.NumberOfEvents
if self.workflow_commons.has_key('Luminosity'):
self.luminosity = self.workflow_commons['Luminosity']
##Additional info: cross section only for the time being, comes from WHIZARD
if self.workflow_commons.has_key('Info'):
if 'stdhepcut' in self.workflow_commons['Info']:
self.sel_eff = self.workflow_commons['Info']['stdhepcut']['Reduction']
self.cut_eff = self.workflow_commons['Info']['stdhepcut']['CutEfficiency']
del self.workflow_commons['Info']['stdhepcut']
self.add_info = DEncode.encode(self.workflow_commons['Info'])
return S_OK('Parameters resolved')
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
switches = {}
archiveLFN = '/vo/tars/myTar.tar'
op.Arguments = DEncode.encode({
'SourceSE':
switches.get('SourceSE', 'SOURCE-SE'),
'TarballSE':
switches.get('TarballSE', 'TARBALL-SE'),
'RegisterDescendent':
False,
'ArchiveLFN':
archiveLFN
})
op.Type = 'ArchiveFiles'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
def _reportFileUsage(self, dirDict):
"""Send the data usage report (site,dirDict) where dirDict = {'Dataset':NumberOfHits}
example: {'/lhcb/certification/test/ALLSTREAMS.DST/00000002/0000/': 1,
'/lhcb/LHCb/Collision11/BHADRON.DST/00012957/0000/': 2}
"""
self.log.verbose('Reporting input file usage:')
for entry in dirDict:
self.log.verbose('%s:%s' % (entry, dirDict[entry]))
# dataUsageClient = RPCClient( 'DataManagement/DataUsage', timeout = 120 )
localSite = gConfig.getValue('/LocalSite/Site', 'UNKNOWN')
try:
localSite = localSite.split('.')[1]
except BaseException:
pass
self.log.verbose('Using Site Name: %s' % (localSite))
if self._enableModule():
usageStatus = self.dataUsageClient.sendDataUsageReport(localSite, dirDict)
if not usageStatus['OK']:
self.log.error('Could not send data usage report, preparing a DISET failover request object')
self.log.verbose(usageStatus['rpcStub'])
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode(usageStatus['rpcStub'])
self.request.addOperation(forwardDISETOp)
self.workflow_commons['Request'] = self.request
else:
self.log.info('Would have attempted to report %s at %s' % (dirDict, localSite))
return S_OK()
return S_OK()
def __listToXML(self, name, aList, indent=0, attributes={}):
""" Utility to convert a list to XML
"""
"""
xml_attributes = ''
xml_elements = []
for element in list:
if type(element) is DictType:
xml_elements.append(self.__dictionaryToXML(name[:-1],element,indent+1))
elif type(value) is ListType:
xml_elements.append(self.__listToXML(name[:-1],element,indent+1))
else:
xml_attributes += ' '*(indent+1)*8+'<%s element_type="leaf"><![CDATA[%s]]></%s>\n' % (name[:-1],str(element),name[:-1])
for attr,value in attributes.items():
xml_attributes += ' '*(indent+1)*8+'<%s element_type="leaf"><![CDATA[%s]]></%s>\n' % (attr,str(value),attr)
out = ' '*indent*8+'<%s element_type="list">\n%s\n' % (name,xml_attributes[:-1])
for el in xml_elements:
out += ' '*indent*8+el
out += ' '*indent*8+'</%s>\n' % name
"""
out = ''
if aList:
den = DEncode.encode(aList)
out += ' ' * indent * 8 + '<%s element_type="list">\n' % (name)
out += ' ' * (
indent + 1
) * 8 + '<EncodedString element_type="leaf"><![CDATA[%s]]></EncodedString>\n' % (
den)
out += ' ' * indent * 8 + '</%s>\n' % name
return out
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = "MyRequest"
op = Operation()
switches = {}
archiveLFN = "/vo/tars/myTar.tar"
op.Arguments = DEncode.encode({
"SourceSE":
switches.get("SourceSE", "SOURCE-SE"),
"TarballSE":
switches.get("TarballSE", "TARBALL-SE"),
"RegisterDescendent":
False,
"ArchiveLFN":
archiveLFN,
})
op.Type = "ArchiveFiles"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def serialize( self ):
if self.__manifest:
manifest = ( self.__manifest.dumpAsCFG(), self.__manifest.isDirty() )
else:
manifest = None
return DEncode.encode( ( self.__jid, self.__cache, self.__jobLog, manifest,
self.__initState, self.__insertIntoTQ, tuple( self.__dirtyKeys ) ) )
def applicationSpecificInputs(self):
if 'Enable' in self.step_commons:
self.enable = self.step_commons['Enable']
if not isinstance(self.enable, bool):
self.log.warn(
'Enable flag set to non-boolean value %s, setting to False'
% self.enable)
self.enable = False
if 'ProductionOutputData' in self.workflow_commons:
self.prodOutputLFNs = self.workflow_commons[
'ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
self.nbofevents = self.NumberOfEvents
if 'Luminosity' in self.workflow_commons:
self.luminosity = self.workflow_commons['Luminosity']
##Additional info: cross section only for the time being, comes from WHIZARD
if 'Info' in self.workflow_commons:
if 'stdhepcut' in self.workflow_commons['Info']:
self.sel_eff = self.workflow_commons['Info']['stdhepcut'][
'Reduction']
self.cut_eff = self.workflow_commons['Info']['stdhepcut'][
'CutEfficiency']
del self.workflow_commons['Info']['stdhepcut']
self.add_info = DEncode.encode(self.workflow_commons['Info'])
return S_OK('Parameters resolved')
def __listToXML(self,name,list,indent = 0):
""" Utility to convert a list to XML
"""
out = ''
if list:
den = DEncode.encode(list)
out += ' '*indent*8+'<%s element_type="list">\n' % (name)
out += ' '*(indent+1)*8+'<EncodedString element_type="leaf"><![CDATA[%s]]></EncodedString>\n' % (den)
out += ' '*indent*8+'</%s>\n' % name
return out
def encode(inData):
""" Encode the input data
:param inData: data to be encoded
:return: an encoded string
"""
if os.getenv('DIRAC_USE_JSON_ENCODE', 'NO').lower() in ('yes', 'true'):
return JEncode.encode(inData)
return DEncode.encode(inData)
def encode(inData):
"""Encode the input data
:param inData: data to be encoded
:return: an encoded string
"""
if os.getenv("DIRAC_USE_JSON_ENCODE", "NO").lower() in ("yes", "true"):
return JEncode.encode(inData)
return DEncode.encode(inData)
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def __executePythonFunction( self, function, writePipe, *stArgs, **stKeyArgs ):
"""
execute function :funtion: using :stArgs: and :stKeyArgs:
"""
try:
os.write( writePipe, DEncode.encode( S_OK( function( *stArgs, **stKeyArgs ) ) ) )
except OSError, x:
if str( x ) == '[Errno 32] Broken pipe':
# the parent has died
pass
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
commitOp = None
if not result['OK']:
# Generate Request
commitOp = Operation()
commitOp.Type = 'SetFileStatus'
commitOp.Arguments = DEncode.encode( {'transformation':self.transformation, 'statusDict':self.statusDict, 'force':self.force} )
return S_OK( commitOp )
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def setUp( self ):
""" test set up """
self.hiArgs = ( ( "RequestManagement/RequestManager",
{ "keepAliveLapse": 10, "timeout": 5 } ),
"foo",
( 12345, { "Hi": "There!" } ) )
self.req = Request( { "RequestName": "testRequest" } )
self.op = Operation( { "Type": "ForwardDISET",
"Arguments": DEncode.encode( self.hiArgs ) } )
self.req += self.op
def __listToXML_new(self, xmldoc, tagName, aList):
# # create
tag = xmldoc.createElement(tagName)
tag.setAttribute("element_type", "list")
encodedStringTag = xmldoc.createElement("EncodedString")
encodedStringTag.setAttribute("element_type", "leaf")
encodedStringCDATA = xmldoc.createCDATASection(DEncode.encode(aList))
# # appending
encodedStringTag.appendChild(encodedStringCDATA)
tag.appendChild(encodedStringTag)
# # returning
return tag
def __listToXML_new( self, xmldoc, tagName, aList ): # # create tag = xmldoc.createElement( tagName ) tag.setAttribute( "element_type", "list" ) encodedStringTag = xmldoc.createElement( "EncodedString" ) encodedStringTag.setAttribute( "element_type", "leaf" ) encodedStringCDATA = xmldoc.createCDATASection( DEncode.encode( aList ) ) # # appending encodedStringTag.appendChild( encodedStringCDATA ) tag.appendChild( encodedStringTag ) # # returning return tag
def codeRequestInFileId(plotRequest, compressIfPossible=True):
compress = compressIfPossible and gZCompressionEnabled
thbStub = False
if compress:
plotStub = "Z:%s" % base64.urlsafe_b64encode(
zlib.compress(DEncode.encode(plotRequest), 9)).decode()
elif not gForceRawEncoding:
plotStub = "S:%s" % base64.urlsafe_b64encode(
DEncode.encode(plotRequest))
else:
plotStub = "R:%s" % DEncode.encode(plotRequest)
# If thumbnail requested, use plot as thumbnail, and generate stub for plot without one
extraArgs = plotRequest['extraArgs']
if 'thumbnail' in extraArgs and extraArgs['thumbnail']:
thbStub = plotStub
extraArgs['thumbnail'] = False
if compress:
plotStub = "Z:%s" % base64.urlsafe_b64encode(
zlib.compress(DEncode.encode(plotRequest), 9)).decode()
elif not gForceRawEncoding:
plotStub = "S:%s" % base64.urlsafe_b64encode(
DEncode.encode(plotRequest)).decode()
else:
plotStub = "R:%s" % DEncode.encode(plotRequest).decode()
return S_OK({'plot': plotStub, 'thumbnail': thbStub})
def web_saveAppState(self):
up = self.__getUP()
try:
name = self.request.arguments['name'][-1]
state = self.request.arguments['state'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
data = base64.b64encode(zlib.compress(DEncode.encode(state), 9))
result = yield self.threadTask(up.storeVar, name, data)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
def __listToXML(self, name, list, indent=0):
""" Utility to convert a list to XML
"""
out = ''
if list:
den = DEncode.encode(list)
out += ' ' * indent * 8 + '<%s element_type="list">\n' % (name)
out += ' ' * (
indent + 1
) * 8 + '<EncodedString element_type="leaf"><![CDATA[%s]]></EncodedString>\n' % (
den)
out += ' ' * indent * 8 + '</%s>\n' % name
return out
def web_saveAppState( self ):
up = self.__getUP()
try:
name = self.request.arguments[ 'name' ][-1]
state = self.request.arguments[ 'state' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
data = base64.b64encode( zlib.compress( DEncode.encode( state ), 9 ) )
result = yield self.threadTask( up.storeVar, name, data )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
self.set_status( 200 )
self.finish()
def generateForwardDISET(self):
""" Commit the accumulated records and generate request eventually
"""
result = self.commit()
commitOp = None
if not result['OK']:
# Generate Request
commitOp = Operation()
commitOp.Type = 'SetFileStatus'
commitOp.Arguments = DEncode.encode({'transformation': self.transformation,
'statusDict': self.statusDict, 'force': self.force})
return S_OK(commitOp)
def __executePythonFunction(self, function, writePipe, *stArgs, **stKeyArgs):
"""
execute function :funtion: using :stArgs: and :stKeyArgs:
"""
from DIRAC.Core.Utilities import DEncode
try:
os.write(writePipe, DEncode.encode(S_OK(function(*stArgs, **stKeyArgs))))
except OSError as x:
if str(x) == '[Errno 32] Broken pipe':
# the parent has died
pass
except Exception as x:
self.log.exception('Exception while executing', function.__name__)
os.write(writePipe, DEncode.encode(S_ERROR(str(x))))
# HACK: Allow some time to flush logs
time.sleep(1)
try:
os.close(writePipe)
finally:
os._exit(0)
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if "FailedResults" in result:
for res in result['FailedResults']:
if 'rpcStub' in res:
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( res['rpcStub'] )
return S_OK( forwardDISETOp )
def serialize(self):
if self.__manifest:
manifest = [self.__manifest.dumpAsCFG(), self.__manifest.isDirty()]
else:
manifest = None
data = DEncode.encode([
self.__jid,
self.__cache,
self.__jobLog,
manifest,
self.__initState,
self.__insertIntoTQ,
list(self.__dirtyKeys),
])
return data.decode()
def generateForwardDISET(self):
"""Commit the accumulated records and generate request eventually"""
result = self.commit()
commitOp = None
if not result["OK"]:
# Generate Request
commitOp = Operation()
commitOp.Type = "SetFileStatus"
commitOp.Arguments = DEncode.encode({
"transformation": self.transformation,
"statusDict": self.statusDict,
"force": self.force
})
return S_OK(commitOp)
def web_saveAppState(self):
self.__tc.setSetup(False)
try:
app = self.request.arguments['app'][-1]
name = self.request.arguments['name'][-1]
state = self.request.arguments['state'][-1]
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
data = base64.b64encode(zlib.compress(DEncode.encode(state), 9))
up = UserProfileClient("Web/App/%s" % app)
result = yield self.threadTask(up.storeVar, name, data)
if not result['OK']:
raise WErr.fromSERROR(result)
self.set_status(200)
self.finish()
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if result.has_key( 'rpcStub' ):
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( result['rpcStub'] )
else:
return S_ERROR( 'Could not create ForwardDISET operation' )
return S_OK( forwardDISETOp )
def _sendToFailover(rpcStub):
"""Create a ForwardDISET operation for failover"""
try:
request = Request()
request.RequestName = "Accounting.DataStore.%s.%s" % (time.time(), random.random())
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode(rpcStub)
request.addOperation(forwardDISETOp)
return ReqClient().putRequest(request)
# We catch all the exceptions, because it should never crash
except Exception as e: # pylint: disable=broad-except
return S_ERROR(ERMSUKN, "Exception sending accounting failover request: %s" % repr(e))
def web_saveAppState( self ):
self.__tc.setSetup( False )
try:
app = self.request.arguments[ 'app' ][-1]
name = self.request.arguments[ 'name' ][-1]
state = self.request.arguments[ 'state' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
data = base64.b64encode( zlib.compress( DEncode.encode( state ), 9 ) )
up = UserProfileClient( "Web/App/%s" % app )
result = yield self.threadTask( up.storeVar, name, data )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
self.set_status( 200 )
self.finish()
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if result.has_key( 'rpcStub' ):
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( result['rpcStub'] )
else:
return S_ERROR( 'Could not create ForwardDISET operation' )
return S_OK( forwardDISETOp )
def setUp(self):
""" test set up """
self.hiArgs = (("RequestManagement/RequestManager", {
"keepAliveLapse": 10,
"timeout": 5
}), "foo", (12345, {
"Hi": "There!"
}))
self.req = Request({"RequestName": "testRequest"})
self.op = Operation({
"Type": "ForwardDISET",
"Arguments": DEncode.encode(self.hiArgs)
})
self.req += self.op
def saveView( self, viewName, viewDescription ):
"""
Save a view in the catalog
"""
if 'stacked' not in viewDescription :
viewDescription[ 'stacked' ] = False
if 'label' not in viewDescription:
viewDescription[ 'label' ] = ""
if 'variable' in viewDescription:
for varField in viewDescription[ 'variable' ]:
if varField in viewDescription[ 'definition' ]:
del( viewDescription[ 'definition' ][ varField ] )
else:
viewDescription[ 'variable' ] = []
acCatalog = self.__createCatalog()
return acCatalog.registerView( viewName, DEncode.encode( viewDescription ), viewDescription[ 'variable' ] )
def saveView(self, viewName, viewDescription):
"""
Save a view in the catalog
"""
if "stacked" not in viewDescription:
viewDescription["stacked"] = False
if "label" not in viewDescription:
viewDescription["label"] = ""
if "variable" in viewDescription:
for varField in viewDescription["variable"]:
if varField in viewDescription["definition"]:
del (viewDescription["definition"][varField])
else:
viewDescription["variable"] = []
acCatalog = self.__createCatalog()
return acCatalog.registerView(viewName, DEncode.encode(viewDescription), viewDescription["variable"])
def __executePythonFunction(self, function, writePipe, *stArgs,
**stKeyArgs):
"""
execute function :funtion: using :stArgs: and :stKeyArgs:
"""
from DIRAC.Core.Utilities import DEncode
try:
os.write(writePipe,
DEncode.encode(S_OK(function(*stArgs, **stKeyArgs))))
except OSError, x:
if str(x) == '[Errno 32] Broken pipe':
# the parent has died
pass
def _sendToFailover( rpcStub ):
""" Create a ForwardDISET operation for failover
"""
try:
request = Request()
request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() )
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( rpcStub )
request.addOperation( forwardDISETOp )
return ReqClient().putRequest( request )
# We catch all the exceptions, because it should never crash
except Exception as e: # pylint: disable=broad-except
return S_ERROR( ERMSUKN, "Exception sending accounting failover request: %s" % repr( e ) )
def initializeDB( self ):
"""
Initializes and creates monitoring catalog db if it doesn't exist
"""
acCatalog = self.__createCatalog()
if not acCatalog.createSchema():
return False
#Register default view if it's not there
viewNames = [ str( v[1] ) for v in acCatalog.getViews( False ) ]
if 'Dynamic component view' in viewNames:
return True
return acCatalog.registerView( "Dynamic component view",
DEncode.encode( { 'variable': ['sources.componentName'],
'definition': {},
'stacked': True,
'groupBy': ['activities.description'],
'label': '$SITE'} ),
['sources.componentName'] )
def saveView(self, viewName, viewDescription):
"""
Save a view in the catalog
"""
if 'stacked' not in viewDescription:
viewDescription['stacked'] = False
if 'label' not in viewDescription:
viewDescription['label'] = ""
if 'variable' in viewDescription:
for varField in viewDescription['variable']:
if varField in viewDescription['definition']:
del (viewDescription['definition'][varField])
else:
viewDescription['variable'] = []
acCatalog = self.__createCatalog()
return acCatalog.registerView(viewName,
DEncode.encode(viewDescription),
viewDescription['variable'])
def __init__(self,rpcStub= None,executionOrder=0):
"""Instantiates the Workflow object and some default parameters.
"""
self.subAttributeNames = ['Status','SubRequestID','Operation','ExecutionOrder','CreationTime','LastUpdate','Arguments']
self.subAttributes = {}
for attr in self.subAttributeNames:
self.subAttributes[attr] = "Unknown"
# Some initial values
self.subAttributes['Status'] = "Waiting"
self.subAttributes['SubRequestID'] = makeGuid()
self.subAttributes['CreationTime'] = Time.toString()
self.subAttributes['ExecutionOrder'] = executionOrder
if rpcStub:
self.subAttributes['Arguments'] = DEncode.encode(rpcStub)
self.subAttributes['Operation'] = rpcStub[1]
def generateForwardDISET( self ):
""" Generate and return failover requests for the operations in the internal cache
"""
fowardDISETOp = None
result = self.sendStoredStatusInfo()
if not result['OK']:
if 'rpcStub' in result:
rpcStub = result['rpcStub']
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( rpcStub )
else:
return S_ERROR( 'Could not create ForwardDISET operation' )
return S_OK( fowardDISETOp )
def __processUpdateAlarmModifications( self, modifications ):
if type( modifications ) != types.DictType:
return S_ERROR( "Modifications must be a dictionary" )
updateFields = []
followers = []
for field in modifications:
if field not in self.__updateAlarmModificableFields:
return S_ERROR( "%s is not a valid modificable field" % field )
value = modifications[ field ]
result = self.__checkAlarmField( field , value )
if not result[ 'OK' ]:
return result
if field == 'assignee':
followers = result[ 'Value' ]
result = self._escapeString( modifications[ field ] )
if not result[ 'OK' ]:
return result
updateFields.append( "%s=%s" % ( field, result[ 'Value' ] ) )
return S_OK( ( ", ".join( updateFields ), DEncode.encode( modifications ), followers ) )
def generateForwardDISET(self):
""" Generate and return failover requests for the operations in the internal cache
"""
fowardDISETOp = None
result = self.sendStoredStatusInfo()
if not result["OK"]:
if "rpcStub" in result:
rpcStub = result["rpcStub"]
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode(rpcStub)
else:
return S_ERROR("Could not create job parameters sub-request")
return S_OK(fowardDISETOp)
def web_changeView( self ):
up = self.__getUP()
try:
desktopName = self.request.arguments[ 'desktop' ][-1]
view = self.request.arguments[ 'view' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
result = yield self.threadTask( up.retrieveVar, desktopName )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
data = result['Value']
oDesktop = json.loads( DEncode.decode( zlib.decompress( base64.b64decode( data ) ) )[0] )
oDesktop[unicode( 'view' )] = unicode( view )
oDesktop = json.dumps( oDesktop )
data = base64.b64encode( zlib.compress( DEncode.encode( oDesktop ), 9 ) )
result = yield self.threadTask( up.storeVar, desktopName, data )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
self.set_status( 200 )
self.finish()
def sendData( self, uData, prefix = False ):
self.__updateLastActionTimestamp()
sCodedData = DEncode.encode( uData )
if prefix:
dataToSend = "%s%s:%s" % ( prefix, len( sCodedData ), sCodedData )
else:
dataToSend = "%s:%s" % ( len( sCodedData ), sCodedData )
for index in range( 0, len( dataToSend ), self.packetSize ):
bytesToSend = len( dataToSend[ index : index + self.packetSize ] )
packSentBytes = 0
while packSentBytes < bytesToSend:
try:
result = self._write( dataToSend[ index + packSentBytes : index + bytesToSend ] )
if not result[ 'OK' ]:
return result
sentBytes = result[ 'Value' ]
except Exception, e:
return S_ERROR( "Exception while sending data: %s" % e )
if sentBytes == 0:
return S_ERROR( "Connection closed by peer" )
packSentBytes += sentBytes
def initializeDB(self):
"""
Initializes and creates monitoring catalog db if it doesn't exist
"""
acCatalog = self.__createCatalog()
if not acCatalog.createSchema():
return False
# Register default view if it's not there
viewNames = [str(v[1]) for v in acCatalog.getViews(False)]
if "Dynamic component view" in viewNames:
return True
return acCatalog.registerView(
"Dynamic component view",
DEncode.encode(
{
"variable": ["sources.componentName"],
"definition": {},
"stacked": True,
"groupBy": ["activities.description"],
"label": "$SITE",
}
),
["sources.componentName"],
)
def web_saveAppState( self ):
up = self.__getUP()
try:
name = self.request.arguments[ 'name' ][-1]
state = self.request.arguments[ 'state' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
data = base64.b64encode( zlib.compress( DEncode.encode( state ), 9 ) )
# before we save the state (modify the state) we have to remeber the actual access: ReadAccess and PublishAccess
result = yield self.threadTask( up.getVarPermissions, name )
if result['OK']:
access = result['Value']
else:
access = {'ReadAccess': 'USER', 'PublishAccess': 'USER'} # this is when the application/desktop does not exists.
result = yield self.threadTask( up.storeVar, name, data )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
# change the access to the application/desktop
result = yield self.threadTask( up.setVarPermissions, name, access )
if not result[ 'OK' ]:
raise WErr.fromSERROR( result )
self.set_status( 200 )
self.finish()
def codeRequestInFileId( plotRequest, compressIfPossible = True ):
compress = compressIfPossible and gZCompressionEnabled
thbStub = False
if compress:
plotStub = "Z:%s" % base64.urlsafe_b64encode( zlib.compress( DEncode.encode( plotRequest ), 9 ) )
elif not gForceRawEncoding:
plotStub = "S:%s" % base64.urlsafe_b64encode( DEncode.encode( plotRequest ) )
else:
plotStub = "R:%s" % DEncode.encode( plotRequest )
#If thumbnail requested, use plot as thumbnail, and generate stub for plot without one
extraArgs = plotRequest[ 'extraArgs' ]
if 'thumbnail' in extraArgs and extraArgs[ 'thumbnail' ]:
thbStub = plotStub
extraArgs[ 'thumbnail' ] = False
if compress:
plotStub = "Z:%s" % base64.urlsafe_b64encode( zlib.compress( DEncode.encode( plotRequest ), 9 ) )
elif not gForceRawEncoding:
plotStub = "S:%s" % base64.urlsafe_b64encode( DEncode.encode( plotRequest ) )
else:
plotStub = "R:%s" % DEncode.encode( plotRequest )
return S_OK( { 'plot' : plotStub, 'thumbnail' : thbStub } )
fromSecs = toSecs - 86400
elif timeLength == "month":
fromSecs = toSecs - 2592000
elif fromSecs == "year":
fromDate = toSecs - 31104000
else:
return S_ERROR( "Time length value not valid" )
else:
fromDate = str( request.params[ 'fromDate' ] )
toDate = str( request.params[ 'toDate' ] )
fromSecs = self.__dateToSecs( fromDate )
toSecs = self.__dateToSecs( toDate )
except Exception, e:
return S_ERROR( "Error while processing plot parameters: %s" % str( e ) )
rpcClient = getRPCClient( "Framework/Monitoring" )
requestStub = DEncode.encode( plotRequest )
retVal = rpcClient.tryView( fromSecs, toSecs, requestStub )
if not retVal[ 'OK' ]:
return retVal
return S_OK( { 'images' : retVal[ 'Value' ], 'stub' : requestStub } )
@jsonify
def saveView( self ):
"""
Save a view
"""
try:
plotRequest = simplejson.loads( request.params[ 'plotRequest' ] )
viewName = str( request.params[ 'viewName' ] )
except Exception, e:
return S_ERROR( "Error while processing plot parameters: %s" % str( e ) )
def newAlarm( self, alarmDef ):
""" Create a new alarm record
"""
followers = []
for field in self.__newAlarmMandatoryFields:
if field not in alarmDef:
return S_ERROR( "Oops. Missing %s" % field )
result = self.__checkAlarmField( field, alarmDef[ field ] )
if not result[ 'OK' ]:
return result
if field == 'assignee':
followers = result[ 'Value' ]
author = alarmDef[ 'author' ]
if author not in followers:
followers.append( author )
sqlFieldsName = []
sqlFieldsValue = []
for field in self.__newAlarmMandatoryFields:
if field == 'notifications':
notifications = {}
for nType in self.__validAlarmNotifications:
if nType in alarmDef[ field ]:
notifications[ nType ] = 1
else:
notifications[ nType ] = 0
val = DEncode.encode( notifications )
else:
val = alarmDef[ field ]
#Add to the list of fields to add
sqlFieldsName.append( field )
result = self._escapeString( val )
if result['OK']:
sqlFieldsValue.append( result['Value'] )
else:
return S_ERROR( 'Failed to escape value %s' % val )
sqlFieldsName.extend( [ 'CreationTime', 'ModTime' ] )
sqlFieldsValue.extend( [ 'UTC_TIMESTAMP()', 'UTC_TIMESTAMP()' ] )
#Get the defined alarmkey and generate a random one if not defined
if 'alarmKey' in alarmDef:
result = self._escapeString( alarmDef[ 'alarmKey' ] )
if result['OK']:
alarmKey = result['Value']
else:
return S_ERROR( 'Failed to escape value %s for key AlarmKey' % val )
gLogger.info( "Checking there are no alarms with key %s" % alarmKey )
result = self._query( "SELECT AlarmId FROM `ntf_Alarms` WHERE AlarmKey=%s" % alarmKey )
if not result[ 'OK' ]:
return result
if result[ 'Value' ]:
return S_ERROR( "Oops, alarm with id %s has the same alarm key!" % result[ 'Value' ][0][0] )
else:
alarmKey = str( time.time() )[-31:]
sqlFieldsName.append( 'AlarmKey' )
sqlFieldsValue.append( alarmKey )
sqlInsert = "INSERT INTO `ntf_Alarms` (%s) VALUES (%s)" % ( ",".join( sqlFieldsName ),
",".join( sqlFieldsValue ) )
result = self._update( sqlInsert )
if not result['OK']:
return result
alarmId = result[ 'lastRowId' ]
for follower in followers:
result = self.modifyFollowerForAlarm( alarmId, follower, notifications )
if not result[ 'OK' ]:
varMsg = "\nFollower: %s\nAlarm: %s\nError: %s" % ( follower, alarmId, result['Message'] )
self.log.error( "Couldn't set follower for alarm", varMsg )
self.__notifyAlarm( alarmId )
return S_OK( alarmId )
