class TransferCommand(Command): ''' Transfer "master" Command ''' def __init__(self, args=None, clients=None): super(TransferCommand, self).__init__(args, clients) if 'ReportsClient' in self.apis: self.rClient = self.apis['ReportsClient'] else: self.rClient = ReportsClient() if 'ReportGenerator' in self.apis: self.rgClient = self.apis['ReportGenerator'] else: self.rgClient = RPCClient('Accounting/ReportGenerator') self.rClient.rpcClient = self.rgClient if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient() def _storeCommand(self, results): ''' Stores the results of doNew method on the database. ''' for result in results: resQuery = self.rmClient.addOrModifyTransferCache( result['SourceName'], result['DestinationName'], result['Metric'], result['Value']) if not resQuery['OK']: return resQuery return S_OK() def _prepareCommand(self): ''' TransferChannelCommand requires four arguments: - hours : <int> - direction : Source | Destination - elementName : <str> - metric : Quality | FailedTransfers GGUSTickets are associated with gocDB names, so we have to transform the diracSiteName into a gocSiteName. ''' if not 'hours' in self.args: return S_ERROR('Number of hours not specified') hours = self.args['hours'] if not 'direction' in self.args: return S_ERROR('direction is missing') direction = self.args['direction'] if direction not in ['Source', 'Destination']: return S_ERROR('direction is not Source nor Destination') if not 'name' in self.args: return S_ERROR('"name" is missing') name = self.args['name'] if not 'metric' in self.args: return S_ERROR('metric is missing') metric = self.args['metric'] if metric not in ['Quality', 'FailedTransfers']: return S_ERROR('metric is not Quality nor FailedTransfers') return S_OK((hours, name, direction, metric)) def doNew(self, masterParams=None): ''' Gets the parameters to run, either from the master method or from its own arguments. For every elementName ( cannot process bulk queries.. ) contacts the accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } } If there are ggus tickets, are recorded and then returned. ''' if masterParams is not None: hours, name, direction, metric = masterParams else: params = self._prepareCommand() if not params['OK']: return params hours, name, direction, metric = params['Value'] toD = datetime.utcnow() fromD = toD - timedelta(hours=hours) # dictionary with conditions for the accounting transferDict = {'OperationType': 'putAndRegister', direction: name} if metric == 'FailedTransfers': transferDict['FinalStatus'] = ['Failed'] transferResults = self.rClient.getReport('DataOperation', metric, fromD, toD, transferDict, 'Channel') if not transferResults['OK']: return transferResults transferResults = transferResults['Value'] if not 'data' in transferResults: return S_ERROR('Missing data key') transferResults = transferResults['data'] uniformResult = [] for channel, elementDict in transferResults.items(): try: source, destination = channel.split(' -> ') except ValueError: continue channelDict = {} channelDict['SourceName'] = source channelDict['DestinationName'] = destination channelDict['Metric'] = metric channelDict['Value'] = sum(elementDict.values()) / len( elementDict.values()) uniformResult.append(channelDict) storeRes = self._storeCommand(uniformResult) if not storeRes['OK']: return storeRes # Compute mean of all transfer channels value = 0 for channelDict in uniformResult: value += channelDict['Value'] if uniformResult: value = float(value) / len(uniformResult) else: value = None return S_OK({'Mean': value, 'Name': name}) def doCache(self): ''' Method that reads the cache table and tries to read from it. It will return a list of dictionaries if there are results. ''' params = self._prepareCommand() if not params['OK']: return params _hours, name, direction, metric = params['Value'] sourceName, destinationName = None, None if direction == 'Source': sourceName = name if direction == 'Destination': destinationName = name result = self.rmClient.selectTransferCache(sourceName, destinationName, metric) if not result['OK']: return result result = [dict(zip(result['Columns'], res)) for res in result['Value']] # Compute mean of all transfer channels value = 0 for channelDict in result: value += channelDict['Value'] if result: value = float(value) / len(result) else: value = None return S_OK({'Mean': value, 'Name': name}) def doMaster(self): ''' Master method, which looks little bit spaguetti code, sorry ! - It gets all Sites. - It gets all StorageElements As there is no bulk query, it compares with what we have on the database. It queries a portion of them. ''' sites = CSHelpers.getSites() if not sites['OK']: return sites sites = sites['Value'] ses = CSHelpers.getStorageElements() if not ses['OK']: return ses ses = ses['Value'] elementNames = sites + ses # sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } ) # if not sourceQuery[ 'OK' ]: # return sourceQuery # sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ] # # sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) ) gLogger.info('Processing %s' % ', '.join(elementNames)) for metric in ['Quality', 'FailedTransfers']: for direction in ['Source', 'Destination']: # 2 hours of window result = self.doNew((2, elementNames, direction, metric)) if not result['OK']: self.metrics['failed'].append(result) return S_OK(self.metrics) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class TransferCommand( Command ): ''' Transfer "master" Command ''' def __init__( self, args = None, clients = None ): super( TransferCommand, self ).__init__( args, clients ) if 'ReportsClient' in self.apis: self.rClient = self.apis[ 'ReportsClient' ] else: self.rClient = ReportsClient() if 'ReportGenerator' in self.apis: self.rgClient = self.apis[ 'ReportGenerator' ] else: self.rgClient = RPCClient( 'Accounting/ReportGenerator' ) self.rClient.rpcClient = self.rgClient if 'ResourceManagementClient' in self.apis: self.rmClient = self.apis[ 'ResourceManagementClient' ] else: self.rmClient = ResourceManagementClient() def _storeCommand( self, results ): ''' Stores the results of doNew method on the database. ''' for result in results: resQuery = self.rmClient.addOrModifyTransferCache( result[ 'SourceName' ], result[ 'DestinationName' ], result[ 'Metric' ], result[ 'Value' ] ) if not resQuery[ 'OK' ]: return resQuery return S_OK() def _prepareCommand( self ): ''' TransferChannelCommand requires four arguments: - hours : <int> - direction : Source | Destination - elementName : <str> - metric : Quality | FailedTransfers GGUSTickets are associated with gocDB names, so we have to transform the diracSiteName into a gocSiteName. ''' if not 'hours' in self.args: return S_ERROR( 'Number of hours not specified' ) hours = self.args[ 'hours' ] if not 'direction' in self.args: return S_ERROR( 'direction is missing' ) direction = self.args[ 'direction' ] if direction not in [ 'Source', 'Destination' ]: return S_ERROR( 'direction is not Source nor Destination' ) if not 'name' in self.args: return S_ERROR( '"name" is missing' ) name = self.args[ 'name' ] if not 'metric' in self.args: return S_ERROR( 'metric is missing' ) metric = self.args[ 'metric' ] if metric not in [ 'Quality', 'FailedTransfers' ]: return S_ERROR( 'metric is not Quality nor FailedTransfers' ) return S_OK( ( hours, name, direction, metric ) ) def doNew( self, masterParams = None ): ''' Gets the parameters to run, either from the master method or from its own arguments. For every elementName ( cannot process bulk queries.. ) contacts the accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } } If there are ggus tickets, are recorded and then returned. ''' if masterParams is not None: hours, name, direction, metric = masterParams else: params = self._prepareCommand() if not params[ 'OK' ]: return params hours, name, direction, metric = params[ 'Value' ] toD = datetime.utcnow() fromD = toD - timedelta( hours = hours ) # dictionary with conditions for the accounting transferDict = { 'OperationType' : 'putAndRegister', direction : name } if metric == 'FailedTransfers': transferDict[ 'FinalStatus' ] = [ 'Failed' ] transferResults = self.rClient.getReport( 'DataOperation', metric, fromD, toD, transferDict, 'Channel' ) if not transferResults[ 'OK' ]: return transferResults transferResults = transferResults[ 'Value' ] if not 'data' in transferResults: return S_ERROR( 'Missing data key' ) transferResults = transferResults[ 'data' ] uniformResult = [] for channel, elementDict in transferResults.items(): try: source, destination = channel.split( ' -> ' ) except ValueError: continue channelDict = {} channelDict[ 'SourceName' ] = source channelDict[ 'DestinationName' ] = destination channelDict[ 'Metric' ] = metric channelDict[ 'Value' ] = sum( elementDict.values() ) / len( elementDict.values() ) uniformResult.append( channelDict ) storeRes = self._storeCommand( uniformResult ) if not storeRes[ 'OK' ]: return storeRes # Compute mean of all transfer channels value = 0 for channelDict in uniformResult: value += channelDict[ 'Value' ] if uniformResult: value = float( value ) / len( uniformResult ) else: value = None return S_OK( { 'Mean' : value, 'Name' : name } ) def doCache( self ): ''' Method that reads the cache table and tries to read from it. It will return a list of dictionaries if there are results. ''' params = self._prepareCommand() if not params[ 'OK' ]: return params _hours, name, direction, metric = params[ 'Value' ] sourceName, destinationName = None, None if direction == 'Source': sourceName = name if direction == 'Destination': destinationName = name result = self.rmClient.selectTransferCache( sourceName, destinationName, metric ) if not result[ 'OK' ]: return result result = [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] # Compute mean of all transfer channels value = 0 for channelDict in result: value += channelDict[ 'Value' ] if result: value = float( value ) / len( result ) else: value = None return S_OK( { 'Mean' : value, 'Name' : name } ) def doMaster( self ): ''' Master method, which looks little bit spaguetti code, sorry ! - It gets all Sites. - It gets all StorageElements As there is no bulk query, it compares with what we have on the database. It queries a portion of them. ''' sites = CSHelpers.getSites() if not sites[ 'OK' ]: return sites sites = sites[ 'Value' ] ses = CSHelpers.getStorageElements() if not ses[ 'OK' ]: return ses ses = ses[ 'Value' ] elementNames = sites + ses # sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } ) # if not sourceQuery[ 'OK' ]: # return sourceQuery # sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ] # # sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) ) gLogger.info( 'Processing %s' % ', '.join( elementNames ) ) for metric in [ 'Quality', 'FailedTransfers' ]: for direction in [ 'Source', 'Destination' ]: # 2 hours of window result = self.doNew( ( 2, elementNames, direction, metric ) ) if not result[ 'OK' ]: self.metrics[ 'failed' ].append( result ) return S_OK( self.metrics ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class TransferCommand(Command): """ Transfer "master" Command """ def __init__(self, args=None, clients=None): super(TransferCommand, self).__init__(args, clients) if "ReportsClient" in self.apis: self.rClient = self.apis["ReportsClient"] else: self.rClient = ReportsClient() if "ResourceManagementClient" in self.apis: self.rmClient = self.apis["ResourceManagementClient"] else: self.rmClient = ResourceManagementClient() def _storeCommand(self, results): """ Stores the results of doNew method on the database. """ for result in results: resQuery = self.rmClient.addOrModifyTransferCache( result["SourceName"], result["DestinationName"], result["Metric"], result["Value"] ) if not resQuery["OK"]: return resQuery return S_OK() def _prepareCommand(self): """ TransferChannelCommand requires four arguments: - hours : <int> - direction : Source | Destination - elementName : <str> - metric : Quality | FailedTransfers GGUSTickets are associated with gocDB names, so we have to transform the diracSiteName into a gocSiteName. """ if "hours" not in self.args: return S_ERROR("Number of hours not specified") hours = self.args["hours"] if "direction" not in self.args: return S_ERROR("direction is missing") direction = self.args["direction"] if direction not in ["Source", "Destination"]: return S_ERROR("direction is not Source nor Destination") if "name" not in self.args: return S_ERROR('"name" is missing') name = self.args["name"] if "metric" not in self.args: return S_ERROR("metric is missing") metric = self.args["metric"] if metric not in ["Quality", "FailedTransfers"]: return S_ERROR("metric is not Quality nor FailedTransfers") return S_OK((hours, name, direction, metric)) def doNew(self, masterParams=None): """ Gets the parameters to run, either from the master method or from its own arguments. For every elementName ( cannot process bulk queries.. ) contacts the accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } } If there are ggus tickets, are recorded and then returned. """ if masterParams is not None: hours, name, direction, metric = masterParams else: params = self._prepareCommand() if not params["OK"]: return params hours, name, direction, metric = params["Value"] toD = datetime.utcnow() fromD = toD - timedelta(hours=hours) # dictionary with conditions for the accounting transferDict = {"OperationType": "putAndRegister", direction: name} if metric == "FailedTransfers": transferDict["FinalStatus"] = ["Failed"] transferResults = self.rClient.getReport("DataOperation", metric, fromD, toD, transferDict, "Channel") if not transferResults["OK"]: return transferResults transferResults = transferResults["Value"] if "data" not in transferResults: return S_ERROR("Missing data key") transferResults = {channel: strToIntDict(value) for channel, value in transferResults["data"].items()} uniformResult = [] for channel, elementDict in transferResults.items(): try: source, destination = channel.split(" -> ") except ValueError: continue channelDict = {} channelDict["SourceName"] = source channelDict["DestinationName"] = destination channelDict["Metric"] = metric channelDict["Value"] = int(sum(elementDict.values()) / len(elementDict.values())) uniformResult.append(channelDict) storeRes = self._storeCommand(uniformResult) if not storeRes["OK"]: return storeRes # Compute mean of all transfer channels value = 0 for channelDict in uniformResult: value += channelDict["Value"] if uniformResult: value = float(value) / len(uniformResult) else: value = None return S_OK({"Mean": value, "Name": name}) def doCache(self): """ Method that reads the cache table and tries to read from it. It will return a list of dictionaries if there are results. """ params = self._prepareCommand() if not params["OK"]: return params _hours, name, direction, metric = params["Value"] sourceName, destinationName = None, None if direction == "Source": sourceName = name if direction == "Destination": destinationName = name result = self.rmClient.selectTransferCache(sourceName, destinationName, metric) if not result["OK"]: return result result = [dict(zip(result["Columns"], res)) for res in result["Value"]] # Compute mean of all transfer channels value = 0 for channelDict in result: value += channelDict["Value"] if result: value = float(value) / len(result) else: value = None return S_OK({"Mean": value, "Name": name}) def doMaster(self): """ Master method, which looks little bit spaguetti code, sorry ! - It gets all Sites. - It gets all StorageElements As there is no bulk query, it compares with what we have on the database. It queries a portion of them. """ sites = getSites() if not sites["OK"]: return sites sites = sites["Value"] elementNames = sites + DMSHelpers().getStorageElements() # sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } ) # if not sourceQuery[ 'OK' ]: # return sourceQuery # sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ] # # sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) ) self.log.info("Processing %s" % ", ".join(elementNames)) for metric in ["Quality", "FailedTransfers"]: for direction in ["Source", "Destination"]: # 2 hours of window result = self.doNew((2, elementNames, direction, metric)) if not result["OK"]: self.metrics["failed"].append(result) return S_OK(self.metrics)