def testUniqueElements(self):
""" uniqueElements tests """
# empty list
aList = []
self.assertEqual(List.uniqueElements(aList), [])
# redundant elements
aList = [1, 1, 2, 3]
self.assertEqual(List.uniqueElements(aList), [1, 2, 3])
def testUniqueElements(self):
""" uniqueElements tests """
# empty list
aList = []
self.assertEqual(List.uniqueElements(aList), [])
# redundant elements
aList = [1, 1, 2, 3]
self.assertEqual(List.uniqueElements(aList), [1, 2, 3])
def __generateTQFindSQL( self, tqDefDict, skipDefinitionCheck = False, connObj = False ):
"""
Find a task queue that has exactly the same requirements
"""
if not skipDefinitionCheck:
tqDefDict = dict( tqDefDict )
result = self._checkTaskQueueDefinition( tqDefDict )
if not result[ 'OK' ]:
return result
tqDefDict = result[ 'Value' ]
sqlCondList = []
for field in self.__singleValueDefFields:
sqlCondList.append( "`tq_TaskQueues`.%s = %s" % ( field, tqDefDict[ field ] ) )
#MAGIC SUBQUERIES TO ENSURE STRICT MATCH
for field in self.__multiValueDefFields:
tableName = '`tq_TQTo%s`' % field
if field in tqDefDict and tqDefDict[ field ]:
firstQuery = "SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId = `tq_TaskQueues`.TQId" % ( tableName, tableName, tableName )
grouping = "GROUP BY %s.TQId" % tableName
valuesList = List.uniqueElements( [ value.strip() for value in tqDefDict[ field ] if value.strip() ] )
numValues = len( valuesList )
secondQuery = "%s AND %s.Value in (%s)" % ( firstQuery, tableName,
",".join( [ "%s" % str( value ) for value in valuesList ] ) )
sqlCondList.append( "%s = (%s %s)" % ( numValues, firstQuery, grouping ) )
sqlCondList.append( "%s = (%s %s)" % ( numValues, secondQuery, grouping ) )
else:
sqlCondList.append( "`tq_TaskQueues`.TQId not in ( SELECT DISTINCT %s.TQId from %s )" % ( tableName, tableName ) )
#END MAGIC: That was easy ;)
return S_OK( " AND ".join( sqlCondList ) )
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
lInitialListOfServers = gatewayList
gLogger.debug("Using configuration gateway",
str(lInitialListOfServers[0]))
else:
lInitialListOfServers = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" %
str(lInitialListOfServers))
lRandomListOfServers = List.randomize(lInitialListOfServers)
gLogger.debug("Randomized server list is %s" %
", ".join(lRandomListOfServers))
for sServer in lRandomListOfServers:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck())
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal['OK']:
return dRetVal
else:
updatingErrorsList.append(dRetVal['Message'])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal['Message']))
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def getSiteMapping(self, resourceType, mapping=''):
""" Get site mapping to resources of a given type
"""
resultDict = {}
if mapping:
result = self.getOptions("SiteTo%sMapping/%s" %
(resourceType, mapping))
if not result['OK']:
return result
for site in result['Value']:
if site != "UseLocalResources":
resultDict[site] = self.getValue(
"SiteTo%sMapping/%s/%s" %
(resourceType, mapping, site), [])
useLocalResources = self.getValue(
"SiteTo%sMapping/%s/UseLocalResources" % (resourceType, mapping),
False)
if useLocalResources:
reHelper = Resources.Resources(vo=self.__vo)
result = reHelper.getEligibleResources(resourceType)
if result['OK']:
for site in result['Value']:
resultDict.setdefault(site, [])
resultDict[site].extend(result['Value'][site])
resultDict[site] = List.uniqueElements(resultDict[site])
return S_OK(resultDict)
def __generateTQFindSQL( self, tqDefDict, skipDefinitionCheck = False, connObj = False ):
"""
Find a task queue that has exactly the same requirements
"""
if not skipDefinitionCheck:
tqDefDict = dict( tqDefDict )
result = self._checkTaskQueueDefinition( tqDefDict )
if not result[ 'OK' ]:
return result
tqDefDict = result[ 'Value' ]
sqlCondList = []
for field in self.__singleValueDefFields:
sqlCondList.append( "`tq_TaskQueues`.%s = %s" % ( field, tqDefDict[ field ] ) )
#MAGIC SUBQUERIES TO ENSURE STRICT MATCH
for field in self.__multiValueDefFields:
tableName = '`tq_TQTo%s`' % field
if field in tqDefDict and tqDefDict[ field ]:
firstQuery = "SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId = `tq_TaskQueues`.TQId" % ( tableName, tableName, tableName )
grouping = "GROUP BY %s.TQId" % tableName
valuesList = List.uniqueElements( [ value.strip() for value in tqDefDict[ field ] if value.strip() ] )
numValues = len( valuesList )
secondQuery = "%s AND %s.Value in (%s)" % ( firstQuery, tableName,
",".join( [ "%s" % str( value ) for value in valuesList ] ) )
sqlCondList.append( "%s = (%s %s)" % ( numValues, firstQuery, grouping ) )
sqlCondList.append( "%s = (%s %s)" % ( numValues, secondQuery, grouping ) )
else:
sqlCondList.append( "`tq_TaskQueues`.TQId not in ( SELECT DISTINCT %s.TQId from %s )" % ( tableName, tableName ) )
#END MAGIC: That was easy ;)
return S_OK( " AND ".join( sqlCondList ) )
def __refresh( self ):
self.__lastUpdateTime = time.time()
gLogger.debug( "Refreshing configuration..." )
gatewayList = getGatewayURLs( "Configuration/Server" )
updatingErrorsList = []
if gatewayList:
lInitialListOfServers = gatewayList
gLogger.debug( "Using configuration gateway", str( lInitialListOfServers[0] ) )
else:
lInitialListOfServers = gConfigurationData.getServers()
gLogger.debug( "Refreshing from list %s" % str( lInitialListOfServers ) )
lRandomListOfServers = List.randomize( lInitialListOfServers )
gLogger.debug( "Randomized server list is %s" % ", ".join( lRandomListOfServers ) )
for sServer in lRandomListOfServers:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient( sServer,
useCertificates = gConfigurationData.useServerCertificate(),
skipCACheck = gConfigurationData.skipCACheck() )
dRetVal = _updateFromRemoteLocation( oClient )
if dRetVal[ 'OK' ]:
return dRetVal
else:
updatingErrorsList.append( dRetVal[ 'Message' ] )
gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % ( sServer, dRetVal[ 'Message' ] ) )
if dRetVal[ 'Message' ].find( "Insane environment" ) > -1:
break
return S_ERROR( "Reason(s):\n\t%s" % "\n\t".join( List.uniqueElements( updatingErrorsList ) ) )
def uniquePath( path = None ):
"""
Utility to squeeze the string containing a PATH-like value to
leave only unique elements preserving the original order
"""
if not isinstance( path, basestring ):
return None
try:
elements = List.uniqueElements( List.fromChar( path, ":" ) )
return ':'.join( elements )
except Exception:
return None
def uniquePath(path=None):
"""
Utility to squeeze the string containing a PATH-like value to
leave only unique elements preserving the original order
"""
if not isinstance(path, six.string_types):
return None
try:
elements = List.uniqueElements(List.fromChar(path, ":"))
return ':'.join(elements)
except Exception:
return None
def _refresh(self, fromMaster=False):
"""
Refresh configuration
"""
self._lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway",
str(initialServerList[0]))
elif fromMaster:
masterServer = gConfigurationData.getMasterServer()
initialServerList = [masterServer]
gLogger.debug("Refreshing from master %s" % masterServer)
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" %
", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient
oClient = ConfigurationClient(
url=sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck(),
)
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal["OK"]:
self._refreshTime = gConfigurationData.getRefreshTime()
return dRetVal
else:
updatingErrorsList.append(dRetVal["Message"])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal["Message"]))
if dRetVal["Message"].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def uniquePath(path=None):
"""
Utility to squeeze the string containing a PATH-like value to
leave only unique elements preserving the original order
"""
if not StringTypes.__contains__(type(path)):
return None
try:
elements = List.uniqueElements(List.fromChar(path, ":"))
return ":".join(elements)
except Exception:
return None
def sync( self ):
gLogger.debug( "Updating configuration internals" )
self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG )
self.remoteServerList = []
localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.localCFG,
disableDangerZones = True )
if localServers:
self.remoteServerList.extend( List.fromChar( localServers, "," ) )
remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.remoteCFG,
disableDangerZones = True )
if remoteServers:
self.remoteServerList.extend( List.fromChar( remoteServers, "," ) )
self.remoteServerList = List.uniqueElements( self.remoteServerList )
self.compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 )
def sync( self ):
gLogger.debug( "Updating configuration internals" )
self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG )
self.remoteServerList = []
localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.localCFG,
disableDangerZones = True )
if localServers:
self.remoteServerList.extend( List.fromChar( localServers, "," ) )
remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
self.remoteCFG,
disableDangerZones = True )
if remoteServers:
self.remoteServerList.extend( List.fromChar( remoteServers, "," ) )
self.remoteServerList = List.uniqueElements( self.remoteServerList )
self.compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 )
def __createTaskQueue( self, tqDefDict, priority = 1, connObj = False ):
"""
Create a task queue
Returns S_OK( tqId ) / S_ERROR
"""
if not connObj:
result = self._getConnection()
if not result[ 'OK' ]:
return S_ERROR( "Can't create task queue: %s" % result[ 'Message' ] )
connObj = result[ 'Value' ]
tqDefDict[ 'CPUTime' ] = self.fitCPUTimeToSegments( tqDefDict[ 'CPUTime' ] )
sqlSingleFields = [ 'TQId', 'Priority' ]
sqlValues = [ "0", str( priority ) ]
for field in self.__singleValueDefFields:
sqlSingleFields.append( field )
sqlValues.append( tqDefDict[ field ] )
#Insert the TQ Disabled
sqlSingleFields.append( "Enabled" )
sqlValues.append( "0" )
cmd = "INSERT INTO tq_TaskQueues ( %s ) VALUES ( %s )" % ( ", ".join( sqlSingleFields ), ", ".join( [ str( v ) for v in sqlValues ] ) )
result = self._update( cmd, conn = connObj )
if not result[ 'OK' ]:
self.log.error( "Can't insert TQ in DB", result[ 'Value' ] )
return result
if 'lastRowId' in result:
tqId = result['lastRowId']
else:
result = self._query( "SELECT LAST_INSERT_ID()", conn = connObj )
if not result[ 'OK' ]:
self.cleanOrphanedTaskQueues( connObj = connObj )
return S_ERROR( "Can't determine task queue id after insertion" )
tqId = result[ 'Value' ][0][0]
for field in self.__multiValueDefFields:
if field not in tqDefDict:
continue
values = List.uniqueElements( [ value for value in tqDefDict[ field ] if value.strip() ] )
if not values:
continue
cmd = "INSERT INTO `tq_TQTo%s` ( TQId, Value ) VALUES " % field
cmd += ", ".join( [ "( %s, %s )" % ( tqId, str( value ) ) for value in values ] )
result = self._update( cmd, conn = connObj )
if not result[ 'OK' ]:
self.log.error( "Failed to insert %s condition" % field, result[ 'Message' ] )
self.cleanOrphanedTaskQueues( connObj = connObj )
return S_ERROR( "Can't insert values %s for field %s: %s" % ( str( values ), field, result[ 'Message' ] ) )
self.log.info( "Created TQ %s" % tqId )
return S_OK( tqId )
def __createTaskQueue( self, tqDefDict, priority = 1, connObj = False ):
"""
Create a task queue
Returns S_OK( tqId ) / S_ERROR
"""
if not connObj:
result = self._getConnection()
if not result[ 'OK' ]:
return S_ERROR( "Can't create task queue: %s" % result[ 'Message' ] )
connObj = result[ 'Value' ]
tqDefDict[ 'CPUTime' ] = self.fitCPUTimeToSegments( tqDefDict[ 'CPUTime' ] )
sqlSingleFields = [ 'TQId', 'Priority' ]
sqlValues = [ "0", str( priority ) ]
for field in self.__singleValueDefFields:
sqlSingleFields.append( field )
sqlValues.append( tqDefDict[ field ] )
#Insert the TQ Disabled
sqlSingleFields.append( "Enabled" )
sqlValues.append( "0" )
cmd = "INSERT INTO tq_TaskQueues ( %s ) VALUES ( %s )" % ( ", ".join( sqlSingleFields ), ", ".join( [ str( v ) for v in sqlValues ] ) )
result = self._update( cmd, conn = connObj )
if not result[ 'OK' ]:
self.log.error( "Can't insert TQ in DB", result[ 'Value' ] )
return result
if 'lastRowId' in result:
tqId = result['lastRowId']
else:
result = self._query( "SELECT LAST_INSERT_ID()", conn = connObj )
if not result[ 'OK' ]:
self.cleanOrphanedTaskQueues( connObj = connObj )
return S_ERROR( "Can't determine task queue id after insertion" )
tqId = result[ 'Value' ][0][0]
for field in self.__multiValueDefFields:
if field not in tqDefDict:
continue
values = List.uniqueElements( [ value for value in tqDefDict[ field ] if value.strip() ] )
if not values:
continue
cmd = "INSERT INTO `tq_TQTo%s` ( TQId, Value ) VALUES " % field
cmd += ", ".join( [ "( %s, %s )" % ( tqId, str( value ) ) for value in values ] )
result = self._update( cmd, conn = connObj )
if not result[ 'OK' ]:
self.log.error( "Failed to insert %s condition" % field, result[ 'Message' ] )
self.cleanOrphanedTaskQueues( connObj = connObj )
return S_ERROR( "Can't insert values %s for field %s: %s" % ( str( values ), field, result[ 'Message' ] ) )
self.log.info( "Created TQ %s" % tqId )
return S_OK( tqId )
def removeJob(self, jobID, sandbox):
""" Remove all the files belonging to the given job
"""
req = "SELECT FileName,FileLink,Partition FROM %s WHERE JobID=%d" % (
sandbox, int(jobID))
result = self._query(req)
if not result['OK']:
return result
if not result['Value']:
return S_OK()
partitions_touched = []
for fname, flink, partition in result['Value']:
if partition:
partitions_touched.append(partition)
req = "DELETE FROM %s WHERE JobID=%d" % (partition, int(jobID))
result = self._update(req)
if not result['OK']:
gLogger.warn('Failed to remove files for job %d' % jobID)
return result
if flink and flink.find('part') == 0:
dummy, pTable, jID, fname = flink[5:].split('/')
if jID == jobID:
partitions_touched.append(partition)
req = "DELETE FROM %s WHERE JobID=%d" % (partition,
int(jobID))
result = self._update(req)
if not result['OK']:
gLogger.warn('Failed to remove files for job %d' %
jobID)
return result
partitions = List.uniqueElements(partitions_touched)
for partition in partitions:
result = self.__updatePartitionSize(sandbox, partition)
req = "DELETE FROM %s WHERE JobID=%d" % (sandbox, int(jobID))
result = self._update(req)
if not result['OK']:
gLogger.warn('Failed to remove files for job %d' % jobID)
return result
gLogger.info('Removed %s files for job %s' % (sandbox, jobID))
return S_OK()
def removeJob(self,jobID,sandbox):
""" Remove all the files belonging to the given job
"""
req = "SELECT FileName,FileLink,Partition FROM %s WHERE JobID=%d" % (sandbox,int(jobID))
result = self._query(req)
if not result['OK']:
return result
if not result['Value']:
return S_OK()
partitions_touched = []
for fname,flink,partition in result['Value']:
if partition:
partitions_touched.append(partition)
req = "DELETE FROM %s WHERE JobID=%d" % (partition,int(jobID))
result = self._update(req)
if not result['OK']:
gLogger.warn('Failed to remove files for job %d' % jobID)
return result
if flink and flink.find('part') == 0:
dummy,pTable,jID,fname = flink[5:].split('/')
if jID == jobID:
partitions_touched.append(partition)
req = "DELETE FROM %s WHERE JobID=%d" % (partition,int(jobID))
result = self._update(req)
if not result['OK']:
gLogger.warn('Failed to remove files for job %d' % jobID)
return result
partitions = List.uniqueElements(partitions_touched)
for partition in partitions:
result = self.__updatePartitionSize(sandbox,partition)
req = "DELETE FROM %s WHERE JobID=%d" % (sandbox,int(jobID))
result = self._update(req)
if not result['OK']:
gLogger.warn('Failed to remove files for job %d' % jobID)
return result
gLogger.info('Removed %s files for job %s' % (sandbox,jobID))
return S_OK()
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway",
str(initialServerList[0]))
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" %
", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck())
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal['OK']:
return dRetVal
else:
updatingErrorsList.append(dRetVal['Message'])
gLogger.warn(
"Can't update from server",
"Error while updating from %s: %s" %
(sServer, dRetVal['Message']))
if dRetVal['Message'].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" %
"\n\t".join(List.uniqueElements(updatingErrorsList)))
def findTaskQueue( self, tqDefDict, skipDefinitionCheck = False, connObj = False ):
"""
Find a task queue that has exactly the same requirements
"""
if not skipDefinitionCheck:
tqDefDict = dict( tqDefDict )
result = self._checkTaskQueueDefinition( tqDefDict )
if not result[ 'OK' ]:
return result
tqDefDict = result[ 'Value' ]
sqlCmd = "SELECT `tq_TaskQueues`.TQId FROM `tq_TaskQueues` WHERE"
sqlCondList = []
for field in self.__singleValueDefFields:
sqlCondList.append( "`tq_TaskQueues`.%s = %s" % ( field, tqDefDict[ field ] ) )
#MAGIC SUBQUERIES TO ENSURE STRICT MATCH
for field in self.__multiValueDefFields:
tableName = '`tq_TQTo%s`' % field
if field in tqDefDict and tqDefDict[ field ]:
firstQuery = "SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId = `tq_TaskQueues`.TQId" % ( tableName, tableName, tableName )
grouping = "GROUP BY %s.TQId" % tableName
valuesList = List.uniqueElements( [ value.strip() for value in tqDefDict[ field ] if value.strip() ] )
numValues = len( valuesList )
secondQuery = "%s AND %s.Value in (%s)" % ( firstQuery, tableName,
",".join( [ "%s" % str( value ) for value in valuesList ] ) )
sqlCondList.append( "%s = (%s %s)" % ( numValues, firstQuery, grouping ) )
sqlCondList.append( "%s = (%s %s)" % ( numValues, secondQuery, grouping ) )
else:
sqlCondList.append( "`tq_TaskQueues`.TQId not in ( SELECT DISTINCT %s.TQId from %s )" % ( tableName, tableName ) )
#END MAGIC: That was easy ;)
sqlCmd = "%s %s" % ( sqlCmd, " AND ".join( sqlCondList ) )
result = self._query( sqlCmd, conn = connObj )
if not result[ 'OK' ]:
return S_ERROR( "Can't find task queue: %s" % result[ 'Message' ] )
data = result[ 'Value' ]
if len( data ) == 0:
return S_OK( { 'found' : False } )
if len( data ) > 1:
gLogger.warn( "Found two task queues for the same requirements", self.__strDict( tqDefDict ) )
return S_OK( { 'found' : True, 'tqId' : data[0][0] } )
def __refresh(self):
self.__lastUpdateTime = time.time()
gLogger.debug("Refreshing configuration...")
gatewayList = getGatewayURLs("Configuration/Server")
updatingErrorsList = []
if gatewayList:
initialServerList = gatewayList
gLogger.debug("Using configuration gateway", str(initialServerList[0]))
else:
initialServerList = gConfigurationData.getServers()
gLogger.debug("Refreshing from list %s" % str(initialServerList))
# If no servers in the initial list, we are supposed to use the local configuration only
if not initialServerList:
return S_OK()
randomServerList = List.randomize(initialServerList)
gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList))
for sServer in randomServerList:
from DIRAC.Core.DISET.RPCClient import RPCClient
oClient = RPCClient(
sServer,
useCertificates=gConfigurationData.useServerCertificate(),
skipCACheck=gConfigurationData.skipCACheck(),
)
dRetVal = _updateFromRemoteLocation(oClient)
if dRetVal["OK"]:
return dRetVal
else:
updatingErrorsList.append(dRetVal["Message"])
gLogger.warn(
"Can't update from server", "Error while updating from %s: %s" % (sServer, dRetVal["Message"])
)
if dRetVal["Message"].find("Insane environment") > -1:
break
return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList)))
def getSiteMapping( self, resourceType, mapping='' ):
""" Get site mapping to resources of a given type
"""
resultDict = {}
if mapping:
result = self.getOptions( "SiteTo%sMapping/%s" % ( resourceType, mapping ) )
if not result['OK']:
return result
for site in result['Value']:
if site != "UseLocalResources":
resultDict[site] = self.getValue( "SiteTo%sMapping/%s/%s" % ( resourceType, mapping, site ), [] )
useLocalResources = self.getValue( "SiteTo%sMapping/%s/UseLocalResources" % ( resourceType, mapping), False )
if useLocalResources:
reHelper = Resources.Resources( vo = self.__vo )
result = reHelper.getEligibleResources( resourceType )
if result['OK']:
for site in result['Value']:
resultDict.setdefault( site, [] )
resultDict[site].extend( result['Value'][site] )
resultDict[site] = List.uniqueElements( resultDict[site] )
return S_OK( resultDict )
def __buildTableList( self, showFieldList ):
""" build the SQL list of tables needed for the query
from the list of variables provided
"""
idPattern = re.compile( r'ID' )
tableDict = { 'MessageTime':'MessageRepository',
'VariableText':'MessageRepository',
'LogLevel':'MessageRepository',
'FixedTextString':'FixedTextMessages',
'ReviewedMessage':'FixedTextMessages',
'SystemName':'Systems', 'SubSystemName':'SubSystems',
'OwnerDN':'UserDNs', 'OwnerGroup':'UserDNs',
'ClientIPNumberString':'ClientIPs',
'ClientFQDN':'ClientIPs', 'SiteName':'Sites'}
tableDictKeys = tableDict.keys()
tableList = []
conjunction = ' NATURAL JOIN '
self.log.debug( '__buildTableList:', 'showFieldList = %s' % showFieldList )
if len( showFieldList ):
for field in showFieldList:
if not idPattern.search( field ) and ( field in tableDictKeys ):
tableList.append( tableDict[field] )
#if re.search( 'MessageTime', ','.join( showFieldList) ):
# tableList.append('MessageRepository')
tableList = List.uniqueElements( tableList )
tableString = ''
try:
tableList.pop( tableList.index( 'MessageRepository' ) )
tableString = 'MessageRepository'
except ValueError:
pass
if tableList.count( 'Sites' ) and tableList.count( 'MessageRepository' ) and not \
tableList.count( 'ClientIPs' ):
tableList.append( 'ClientIPs' )
if tableList.count( 'MessageRepository' ) and tableList.count( 'SubSystems' ) \
and not tableList.count( 'FixedTextMessages' ) and not tableList.count( 'Systems' ):
tableList.append( 'FixedTextMessages' )
tableList.append( 'Systems' )
if tableList.count( 'MessageRepository' ) and tableList.count( 'Systems' ) \
and not tableList.count( 'FixedTextMessages' ):
tableList.append( 'FixedTextMessages' )
if tableList.count( 'FixedTextMessages' ) and tableList.count( 'SubSystems' ) \
and not tableList.count( 'Systems' ):
tableList.append( 'Systems' )
if tableList.count( 'MessageRepository' ) or ( tableList.count( 'FixedTextMessages' ) \
+ tableList.count( 'ClientIPs' ) + tableList.count( 'UserDNs' ) > 1 ) :
tableString = 'MessageRepository'
if tableString and len( tableList ):
tableString = '%s%s' % ( tableString, conjunction )
tableString = '%s%s' % ( tableString,
conjunction.join( tableList ) )
else:
tableString = conjunction.join( List.uniqueElements( tableDict.values() ) )
self.log.debug( '__buildTableList:', 'tableString = "%s"' % tableString )
return tableString
def __buildTableList(self, showFieldList):
""" build the SQL list of tables needed for the query
from the list of variables provided
"""
idPattern = re.compile(r'ID')
tableDict = {
'MessageTime': 'MessageRepository',
'VariableText': 'MessageRepository',
'LogLevel': 'MessageRepository',
'FixedTextString': 'FixedTextMessages',
'ReviewedMessage': 'FixedTextMessages',
'SystemName': 'Systems',
'SubSystemName': 'SubSystems',
'OwnerDN': 'UserDNs',
'OwnerGroup': 'UserDNs',
'ClientIPNumberString': 'ClientIPs',
'ClientFQDN': 'ClientIPs',
'SiteName': 'Sites'
}
tableDictKeys = tableDict.keys()
tableList = []
conjunction = ' NATURAL JOIN '
self.log.debug('__buildTableList:',
'showFieldList = %s' % showFieldList)
if len(showFieldList):
for field in showFieldList:
if not idPattern.search(field) and (field in tableDictKeys):
tableList.append(tableDict[field])
#if re.search( 'MessageTime', ','.join( showFieldList) ):
# tableList.append('MessageRepository')
tableList = List.uniqueElements(tableList)
tableString = ''
try:
tableList.pop(tableList.index('MessageRepository'))
tableString = 'MessageRepository'
except ValueError:
pass
if tableList.count( 'Sites' ) and tableList.count( 'MessageRepository' ) and not \
tableList.count( 'ClientIPs' ):
tableList.append('ClientIPs')
if tableList.count( 'MessageRepository' ) and tableList.count( 'SubSystems' ) \
and not tableList.count( 'FixedTextMessages' ) and not tableList.count( 'Systems' ):
tableList.append('FixedTextMessages')
tableList.append('Systems')
if tableList.count( 'MessageRepository' ) and tableList.count( 'Systems' ) \
and not tableList.count( 'FixedTextMessages' ):
tableList.append('FixedTextMessages')
if tableList.count( 'FixedTextMessages' ) and tableList.count( 'SubSystems' ) \
and not tableList.count( 'Systems' ):
tableList.append('Systems')
if tableList.count( 'MessageRepository' ) or ( tableList.count( 'FixedTextMessages' ) \
+ tableList.count( 'ClientIPs' ) + tableList.count( 'UserDNs' ) > 1 ) :
tableString = 'MessageRepository'
if tableString and len(tableList):
tableString = '%s%s' % (tableString, conjunction)
tableString = '%s%s' % (tableString, conjunction.join(tableList))
else:
tableString = conjunction.join(
List.uniqueElements(tableDict.values()))
self.log.debug('__buildTableList:', 'tableString = "%s"' % tableString)
return tableString
