def testUniqueElements(self): """ uniqueElements tests """ # empty list aList = [] self.assertEqual(List.uniqueElements(aList), []) # redundant elements aList = [1, 1, 2, 3] self.assertEqual(List.uniqueElements(aList), [1, 2, 3])
def __generateTQFindSQL( self, tqDefDict, skipDefinitionCheck = False, connObj = False ): """ Find a task queue that has exactly the same requirements """ if not skipDefinitionCheck: tqDefDict = dict( tqDefDict ) result = self._checkTaskQueueDefinition( tqDefDict ) if not result[ 'OK' ]: return result tqDefDict = result[ 'Value' ] sqlCondList = [] for field in self.__singleValueDefFields: sqlCondList.append( "`tq_TaskQueues`.%s = %s" % ( field, tqDefDict[ field ] ) ) #MAGIC SUBQUERIES TO ENSURE STRICT MATCH for field in self.__multiValueDefFields: tableName = '`tq_TQTo%s`' % field if field in tqDefDict and tqDefDict[ field ]: firstQuery = "SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId = `tq_TaskQueues`.TQId" % ( tableName, tableName, tableName ) grouping = "GROUP BY %s.TQId" % tableName valuesList = List.uniqueElements( [ value.strip() for value in tqDefDict[ field ] if value.strip() ] ) numValues = len( valuesList ) secondQuery = "%s AND %s.Value in (%s)" % ( firstQuery, tableName, ",".join( [ "%s" % str( value ) for value in valuesList ] ) ) sqlCondList.append( "%s = (%s %s)" % ( numValues, firstQuery, grouping ) ) sqlCondList.append( "%s = (%s %s)" % ( numValues, secondQuery, grouping ) ) else: sqlCondList.append( "`tq_TaskQueues`.TQId not in ( SELECT DISTINCT %s.TQId from %s )" % ( tableName, tableName ) ) #END MAGIC: That was easy ;) return S_OK( " AND ".join( sqlCondList ) )
def __refresh(self): self.__lastUpdateTime = time.time() gLogger.debug("Refreshing configuration...") gatewayList = getGatewayURLs("Configuration/Server") updatingErrorsList = [] if gatewayList: lInitialListOfServers = gatewayList gLogger.debug("Using configuration gateway", str(lInitialListOfServers[0])) else: lInitialListOfServers = gConfigurationData.getServers() gLogger.debug("Refreshing from list %s" % str(lInitialListOfServers)) lRandomListOfServers = List.randomize(lInitialListOfServers) gLogger.debug("Randomized server list is %s" % ", ".join(lRandomListOfServers)) for sServer in lRandomListOfServers: from DIRAC.Core.DISET.RPCClient import RPCClient oClient = RPCClient( sServer, useCertificates=gConfigurationData.useServerCertificate(), skipCACheck=gConfigurationData.skipCACheck()) dRetVal = _updateFromRemoteLocation(oClient) if dRetVal['OK']: return dRetVal else: updatingErrorsList.append(dRetVal['Message']) gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % (sServer, dRetVal['Message'])) return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList)))
def getSiteMapping(self, resourceType, mapping=''): """ Get site mapping to resources of a given type """ resultDict = {} if mapping: result = self.getOptions("SiteTo%sMapping/%s" % (resourceType, mapping)) if not result['OK']: return result for site in result['Value']: if site != "UseLocalResources": resultDict[site] = self.getValue( "SiteTo%sMapping/%s/%s" % (resourceType, mapping, site), []) useLocalResources = self.getValue( "SiteTo%sMapping/%s/UseLocalResources" % (resourceType, mapping), False) if useLocalResources: reHelper = Resources.Resources(vo=self.__vo) result = reHelper.getEligibleResources(resourceType) if result['OK']: for site in result['Value']: resultDict.setdefault(site, []) resultDict[site].extend(result['Value'][site]) resultDict[site] = List.uniqueElements(resultDict[site]) return S_OK(resultDict)
def __refresh( self ): self.__lastUpdateTime = time.time() gLogger.debug( "Refreshing configuration..." ) gatewayList = getGatewayURLs( "Configuration/Server" ) updatingErrorsList = [] if gatewayList: lInitialListOfServers = gatewayList gLogger.debug( "Using configuration gateway", str( lInitialListOfServers[0] ) ) else: lInitialListOfServers = gConfigurationData.getServers() gLogger.debug( "Refreshing from list %s" % str( lInitialListOfServers ) ) lRandomListOfServers = List.randomize( lInitialListOfServers ) gLogger.debug( "Randomized server list is %s" % ", ".join( lRandomListOfServers ) ) for sServer in lRandomListOfServers: from DIRAC.Core.DISET.RPCClient import RPCClient oClient = RPCClient( sServer, useCertificates = gConfigurationData.useServerCertificate(), skipCACheck = gConfigurationData.skipCACheck() ) dRetVal = _updateFromRemoteLocation( oClient ) if dRetVal[ 'OK' ]: return dRetVal else: updatingErrorsList.append( dRetVal[ 'Message' ] ) gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % ( sServer, dRetVal[ 'Message' ] ) ) if dRetVal[ 'Message' ].find( "Insane environment" ) > -1: break return S_ERROR( "Reason(s):\n\t%s" % "\n\t".join( List.uniqueElements( updatingErrorsList ) ) )
def uniquePath( path = None ): """ Utility to squeeze the string containing a PATH-like value to leave only unique elements preserving the original order """ if not isinstance( path, basestring ): return None try: elements = List.uniqueElements( List.fromChar( path, ":" ) ) return ':'.join( elements ) except Exception: return None
def uniquePath(path=None): """ Utility to squeeze the string containing a PATH-like value to leave only unique elements preserving the original order """ if not isinstance(path, six.string_types): return None try: elements = List.uniqueElements(List.fromChar(path, ":")) return ':'.join(elements) except Exception: return None
def _refresh(self, fromMaster=False): """ Refresh configuration """ self._lastUpdateTime = time.time() gLogger.debug("Refreshing configuration...") gatewayList = getGatewayURLs("Configuration/Server") updatingErrorsList = [] if gatewayList: initialServerList = gatewayList gLogger.debug("Using configuration gateway", str(initialServerList[0])) elif fromMaster: masterServer = gConfigurationData.getMasterServer() initialServerList = [masterServer] gLogger.debug("Refreshing from master %s" % masterServer) else: initialServerList = gConfigurationData.getServers() gLogger.debug("Refreshing from list %s" % str(initialServerList)) # If no servers in the initial list, we are supposed to use the local configuration only if not initialServerList: return S_OK() randomServerList = List.randomize(initialServerList) gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList)) for sServer in randomServerList: from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient oClient = ConfigurationClient( url=sServer, useCertificates=gConfigurationData.useServerCertificate(), skipCACheck=gConfigurationData.skipCACheck(), ) dRetVal = _updateFromRemoteLocation(oClient) if dRetVal["OK"]: self._refreshTime = gConfigurationData.getRefreshTime() return dRetVal else: updatingErrorsList.append(dRetVal["Message"]) gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % (sServer, dRetVal["Message"])) if dRetVal["Message"].find("Insane environment") > -1: break return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList)))
def uniquePath(path=None): """ Utility to squeeze the string containing a PATH-like value to leave only unique elements preserving the original order """ if not StringTypes.__contains__(type(path)): return None try: elements = List.uniqueElements(List.fromChar(path, ":")) return ":".join(elements) except Exception: return None
def sync( self ): gLogger.debug( "Updating configuration internals" ) self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG ) self.remoteServerList = [] localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath, self.localCFG, disableDangerZones = True ) if localServers: self.remoteServerList.extend( List.fromChar( localServers, "," ) ) remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath, self.remoteCFG, disableDangerZones = True ) if remoteServers: self.remoteServerList.extend( List.fromChar( remoteServers, "," ) ) self.remoteServerList = List.uniqueElements( self.remoteServerList ) self.compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 )
def __createTaskQueue( self, tqDefDict, priority = 1, connObj = False ): """ Create a task queue Returns S_OK( tqId ) / S_ERROR """ if not connObj: result = self._getConnection() if not result[ 'OK' ]: return S_ERROR( "Can't create task queue: %s" % result[ 'Message' ] ) connObj = result[ 'Value' ] tqDefDict[ 'CPUTime' ] = self.fitCPUTimeToSegments( tqDefDict[ 'CPUTime' ] ) sqlSingleFields = [ 'TQId', 'Priority' ] sqlValues = [ "0", str( priority ) ] for field in self.__singleValueDefFields: sqlSingleFields.append( field ) sqlValues.append( tqDefDict[ field ] ) #Insert the TQ Disabled sqlSingleFields.append( "Enabled" ) sqlValues.append( "0" ) cmd = "INSERT INTO tq_TaskQueues ( %s ) VALUES ( %s )" % ( ", ".join( sqlSingleFields ), ", ".join( [ str( v ) for v in sqlValues ] ) ) result = self._update( cmd, conn = connObj ) if not result[ 'OK' ]: self.log.error( "Can't insert TQ in DB", result[ 'Value' ] ) return result if 'lastRowId' in result: tqId = result['lastRowId'] else: result = self._query( "SELECT LAST_INSERT_ID()", conn = connObj ) if not result[ 'OK' ]: self.cleanOrphanedTaskQueues( connObj = connObj ) return S_ERROR( "Can't determine task queue id after insertion" ) tqId = result[ 'Value' ][0][0] for field in self.__multiValueDefFields: if field not in tqDefDict: continue values = List.uniqueElements( [ value for value in tqDefDict[ field ] if value.strip() ] ) if not values: continue cmd = "INSERT INTO `tq_TQTo%s` ( TQId, Value ) VALUES " % field cmd += ", ".join( [ "( %s, %s )" % ( tqId, str( value ) ) for value in values ] ) result = self._update( cmd, conn = connObj ) if not result[ 'OK' ]: self.log.error( "Failed to insert %s condition" % field, result[ 'Message' ] ) self.cleanOrphanedTaskQueues( connObj = connObj ) return S_ERROR( "Can't insert values %s for field %s: %s" % ( str( values ), field, result[ 'Message' ] ) ) self.log.info( "Created TQ %s" % tqId ) return S_OK( tqId )
def removeJob(self, jobID, sandbox): """ Remove all the files belonging to the given job """ req = "SELECT FileName,FileLink,Partition FROM %s WHERE JobID=%d" % ( sandbox, int(jobID)) result = self._query(req) if not result['OK']: return result if not result['Value']: return S_OK() partitions_touched = [] for fname, flink, partition in result['Value']: if partition: partitions_touched.append(partition) req = "DELETE FROM %s WHERE JobID=%d" % (partition, int(jobID)) result = self._update(req) if not result['OK']: gLogger.warn('Failed to remove files for job %d' % jobID) return result if flink and flink.find('part') == 0: dummy, pTable, jID, fname = flink[5:].split('/') if jID == jobID: partitions_touched.append(partition) req = "DELETE FROM %s WHERE JobID=%d" % (partition, int(jobID)) result = self._update(req) if not result['OK']: gLogger.warn('Failed to remove files for job %d' % jobID) return result partitions = List.uniqueElements(partitions_touched) for partition in partitions: result = self.__updatePartitionSize(sandbox, partition) req = "DELETE FROM %s WHERE JobID=%d" % (sandbox, int(jobID)) result = self._update(req) if not result['OK']: gLogger.warn('Failed to remove files for job %d' % jobID) return result gLogger.info('Removed %s files for job %s' % (sandbox, jobID)) return S_OK()
def removeJob(self,jobID,sandbox): """ Remove all the files belonging to the given job """ req = "SELECT FileName,FileLink,Partition FROM %s WHERE JobID=%d" % (sandbox,int(jobID)) result = self._query(req) if not result['OK']: return result if not result['Value']: return S_OK() partitions_touched = [] for fname,flink,partition in result['Value']: if partition: partitions_touched.append(partition) req = "DELETE FROM %s WHERE JobID=%d" % (partition,int(jobID)) result = self._update(req) if not result['OK']: gLogger.warn('Failed to remove files for job %d' % jobID) return result if flink and flink.find('part') == 0: dummy,pTable,jID,fname = flink[5:].split('/') if jID == jobID: partitions_touched.append(partition) req = "DELETE FROM %s WHERE JobID=%d" % (partition,int(jobID)) result = self._update(req) if not result['OK']: gLogger.warn('Failed to remove files for job %d' % jobID) return result partitions = List.uniqueElements(partitions_touched) for partition in partitions: result = self.__updatePartitionSize(sandbox,partition) req = "DELETE FROM %s WHERE JobID=%d" % (sandbox,int(jobID)) result = self._update(req) if not result['OK']: gLogger.warn('Failed to remove files for job %d' % jobID) return result gLogger.info('Removed %s files for job %s' % (sandbox,jobID)) return S_OK()
def __refresh(self): self.__lastUpdateTime = time.time() gLogger.debug("Refreshing configuration...") gatewayList = getGatewayURLs("Configuration/Server") updatingErrorsList = [] if gatewayList: initialServerList = gatewayList gLogger.debug("Using configuration gateway", str(initialServerList[0])) else: initialServerList = gConfigurationData.getServers() gLogger.debug("Refreshing from list %s" % str(initialServerList)) # If no servers in the initial list, we are supposed to use the local configuration only if not initialServerList: return S_OK() randomServerList = List.randomize(initialServerList) gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList)) for sServer in randomServerList: from DIRAC.Core.DISET.RPCClient import RPCClient oClient = RPCClient( sServer, useCertificates=gConfigurationData.useServerCertificate(), skipCACheck=gConfigurationData.skipCACheck()) dRetVal = _updateFromRemoteLocation(oClient) if dRetVal['OK']: return dRetVal else: updatingErrorsList.append(dRetVal['Message']) gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % (sServer, dRetVal['Message'])) if dRetVal['Message'].find("Insane environment") > -1: break return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList)))
def findTaskQueue( self, tqDefDict, skipDefinitionCheck = False, connObj = False ): """ Find a task queue that has exactly the same requirements """ if not skipDefinitionCheck: tqDefDict = dict( tqDefDict ) result = self._checkTaskQueueDefinition( tqDefDict ) if not result[ 'OK' ]: return result tqDefDict = result[ 'Value' ] sqlCmd = "SELECT `tq_TaskQueues`.TQId FROM `tq_TaskQueues` WHERE" sqlCondList = [] for field in self.__singleValueDefFields: sqlCondList.append( "`tq_TaskQueues`.%s = %s" % ( field, tqDefDict[ field ] ) ) #MAGIC SUBQUERIES TO ENSURE STRICT MATCH for field in self.__multiValueDefFields: tableName = '`tq_TQTo%s`' % field if field in tqDefDict and tqDefDict[ field ]: firstQuery = "SELECT COUNT(%s.Value) FROM %s WHERE %s.TQId = `tq_TaskQueues`.TQId" % ( tableName, tableName, tableName ) grouping = "GROUP BY %s.TQId" % tableName valuesList = List.uniqueElements( [ value.strip() for value in tqDefDict[ field ] if value.strip() ] ) numValues = len( valuesList ) secondQuery = "%s AND %s.Value in (%s)" % ( firstQuery, tableName, ",".join( [ "%s" % str( value ) for value in valuesList ] ) ) sqlCondList.append( "%s = (%s %s)" % ( numValues, firstQuery, grouping ) ) sqlCondList.append( "%s = (%s %s)" % ( numValues, secondQuery, grouping ) ) else: sqlCondList.append( "`tq_TaskQueues`.TQId not in ( SELECT DISTINCT %s.TQId from %s )" % ( tableName, tableName ) ) #END MAGIC: That was easy ;) sqlCmd = "%s %s" % ( sqlCmd, " AND ".join( sqlCondList ) ) result = self._query( sqlCmd, conn = connObj ) if not result[ 'OK' ]: return S_ERROR( "Can't find task queue: %s" % result[ 'Message' ] ) data = result[ 'Value' ] if len( data ) == 0: return S_OK( { 'found' : False } ) if len( data ) > 1: gLogger.warn( "Found two task queues for the same requirements", self.__strDict( tqDefDict ) ) return S_OK( { 'found' : True, 'tqId' : data[0][0] } )
def __refresh(self): self.__lastUpdateTime = time.time() gLogger.debug("Refreshing configuration...") gatewayList = getGatewayURLs("Configuration/Server") updatingErrorsList = [] if gatewayList: initialServerList = gatewayList gLogger.debug("Using configuration gateway", str(initialServerList[0])) else: initialServerList = gConfigurationData.getServers() gLogger.debug("Refreshing from list %s" % str(initialServerList)) # If no servers in the initial list, we are supposed to use the local configuration only if not initialServerList: return S_OK() randomServerList = List.randomize(initialServerList) gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList)) for sServer in randomServerList: from DIRAC.Core.DISET.RPCClient import RPCClient oClient = RPCClient( sServer, useCertificates=gConfigurationData.useServerCertificate(), skipCACheck=gConfigurationData.skipCACheck(), ) dRetVal = _updateFromRemoteLocation(oClient) if dRetVal["OK"]: return dRetVal else: updatingErrorsList.append(dRetVal["Message"]) gLogger.warn( "Can't update from server", "Error while updating from %s: %s" % (sServer, dRetVal["Message"]) ) if dRetVal["Message"].find("Insane environment") > -1: break return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList)))
def getSiteMapping( self, resourceType, mapping='' ): """ Get site mapping to resources of a given type """ resultDict = {} if mapping: result = self.getOptions( "SiteTo%sMapping/%s" % ( resourceType, mapping ) ) if not result['OK']: return result for site in result['Value']: if site != "UseLocalResources": resultDict[site] = self.getValue( "SiteTo%sMapping/%s/%s" % ( resourceType, mapping, site ), [] ) useLocalResources = self.getValue( "SiteTo%sMapping/%s/UseLocalResources" % ( resourceType, mapping), False ) if useLocalResources: reHelper = Resources.Resources( vo = self.__vo ) result = reHelper.getEligibleResources( resourceType ) if result['OK']: for site in result['Value']: resultDict.setdefault( site, [] ) resultDict[site].extend( result['Value'][site] ) resultDict[site] = List.uniqueElements( resultDict[site] ) return S_OK( resultDict )
def __buildTableList( self, showFieldList ): """ build the SQL list of tables needed for the query from the list of variables provided """ idPattern = re.compile( r'ID' ) tableDict = { 'MessageTime':'MessageRepository', 'VariableText':'MessageRepository', 'LogLevel':'MessageRepository', 'FixedTextString':'FixedTextMessages', 'ReviewedMessage':'FixedTextMessages', 'SystemName':'Systems', 'SubSystemName':'SubSystems', 'OwnerDN':'UserDNs', 'OwnerGroup':'UserDNs', 'ClientIPNumberString':'ClientIPs', 'ClientFQDN':'ClientIPs', 'SiteName':'Sites'} tableDictKeys = tableDict.keys() tableList = [] conjunction = ' NATURAL JOIN ' self.log.debug( '__buildTableList:', 'showFieldList = %s' % showFieldList ) if len( showFieldList ): for field in showFieldList: if not idPattern.search( field ) and ( field in tableDictKeys ): tableList.append( tableDict[field] ) #if re.search( 'MessageTime', ','.join( showFieldList) ): # tableList.append('MessageRepository') tableList = List.uniqueElements( tableList ) tableString = '' try: tableList.pop( tableList.index( 'MessageRepository' ) ) tableString = 'MessageRepository' except ValueError: pass if tableList.count( 'Sites' ) and tableList.count( 'MessageRepository' ) and not \ tableList.count( 'ClientIPs' ): tableList.append( 'ClientIPs' ) if tableList.count( 'MessageRepository' ) and tableList.count( 'SubSystems' ) \ and not tableList.count( 'FixedTextMessages' ) and not tableList.count( 'Systems' ): tableList.append( 'FixedTextMessages' ) tableList.append( 'Systems' ) if tableList.count( 'MessageRepository' ) and tableList.count( 'Systems' ) \ and not tableList.count( 'FixedTextMessages' ): tableList.append( 'FixedTextMessages' ) if tableList.count( 'FixedTextMessages' ) and tableList.count( 'SubSystems' ) \ and not tableList.count( 'Systems' ): tableList.append( 'Systems' ) if tableList.count( 'MessageRepository' ) or ( tableList.count( 'FixedTextMessages' ) \ + tableList.count( 'ClientIPs' ) + tableList.count( 'UserDNs' ) > 1 ) : tableString = 'MessageRepository' if tableString and len( tableList ): tableString = '%s%s' % ( tableString, conjunction ) tableString = '%s%s' % ( tableString, conjunction.join( tableList ) ) else: tableString = conjunction.join( List.uniqueElements( tableDict.values() ) ) self.log.debug( '__buildTableList:', 'tableString = "%s"' % tableString ) return tableString
def __buildTableList(self, showFieldList): """ build the SQL list of tables needed for the query from the list of variables provided """ idPattern = re.compile(r'ID') tableDict = { 'MessageTime': 'MessageRepository', 'VariableText': 'MessageRepository', 'LogLevel': 'MessageRepository', 'FixedTextString': 'FixedTextMessages', 'ReviewedMessage': 'FixedTextMessages', 'SystemName': 'Systems', 'SubSystemName': 'SubSystems', 'OwnerDN': 'UserDNs', 'OwnerGroup': 'UserDNs', 'ClientIPNumberString': 'ClientIPs', 'ClientFQDN': 'ClientIPs', 'SiteName': 'Sites' } tableDictKeys = tableDict.keys() tableList = [] conjunction = ' NATURAL JOIN ' self.log.debug('__buildTableList:', 'showFieldList = %s' % showFieldList) if len(showFieldList): for field in showFieldList: if not idPattern.search(field) and (field in tableDictKeys): tableList.append(tableDict[field]) #if re.search( 'MessageTime', ','.join( showFieldList) ): # tableList.append('MessageRepository') tableList = List.uniqueElements(tableList) tableString = '' try: tableList.pop(tableList.index('MessageRepository')) tableString = 'MessageRepository' except ValueError: pass if tableList.count( 'Sites' ) and tableList.count( 'MessageRepository' ) and not \ tableList.count( 'ClientIPs' ): tableList.append('ClientIPs') if tableList.count( 'MessageRepository' ) and tableList.count( 'SubSystems' ) \ and not tableList.count( 'FixedTextMessages' ) and not tableList.count( 'Systems' ): tableList.append('FixedTextMessages') tableList.append('Systems') if tableList.count( 'MessageRepository' ) and tableList.count( 'Systems' ) \ and not tableList.count( 'FixedTextMessages' ): tableList.append('FixedTextMessages') if tableList.count( 'FixedTextMessages' ) and tableList.count( 'SubSystems' ) \ and not tableList.count( 'Systems' ): tableList.append('Systems') if tableList.count( 'MessageRepository' ) or ( tableList.count( 'FixedTextMessages' ) \ + tableList.count( 'ClientIPs' ) + tableList.count( 'UserDNs' ) > 1 ) : tableString = 'MessageRepository' if tableString and len(tableList): tableString = '%s%s' % (tableString, conjunction) tableString = '%s%s' % (tableString, conjunction.join(tableList)) else: tableString = conjunction.join( List.uniqueElements(tableDict.values())) self.log.debug('__buildTableList:', 'tableString = "%s"' % tableString) return tableString