def _Broadcast( self ): """ This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs. """ if not self.params: return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." ) targetseParam = self.params['TargetSE'] targetSEs = [] sourceSEs = eval( self.params['SourceSE'] ) if targetseParam.count( '[' ): targetSEs = eval( targetseParam ) elif isinstance( targetseParam, list ): targetSEs = targetseParam else: targetSEs = [targetseParam] # sourceSEs = eval(self.params['SourceSE']) # targetSEs = eval(self.params['TargetSE']) destinations = int( self.params.get( 'Destinations', 0 ) ) if destinations and ( destinations >= len( targetSEs ) ): destinations = 0 status = self.params['Status'] groupSize = self.params['GroupSize'] # Number of files per tasks fileGroups = getFileGroups( self.data ) # groups by SE targetSELfns = {} for replicaSE, lfns in fileGroups.items(): ses = replicaSE.split( ',' ) # sourceSites = self._getSitesForSEs(ses) atSource = False for se in ses: if se in sourceSEs: atSource = True if not atSource: continue for lfn in lfns: targets = [] sources = self._getSitesForSEs( ses ) random.shuffle( targetSEs ) for targetSE in targetSEs: site = self._getSiteForSE( targetSE )['Value'] if not site in sources: if ( destinations ) and ( len( targets ) >= destinations ): continue sources.append( site ) targets.append( targetSE ) # after all, if someone wants to copy to the source, it's his choice strTargetSEs = str.join( ',', sorted( targets ) ) if not targetSELfns.has_key( strTargetSEs ): targetSELfns[strTargetSEs] = [] targetSELfns[strTargetSEs].append( lfn ) tasks = [] for ses, lfns in targetSELfns.items(): tasksLfns = breakListIntoChunks( lfns, groupSize ) for taskLfns in tasksLfns: if ( status == 'Flush' ) or ( len( taskLfns ) >= int( groupSize ) ): # do not allow groups smaller than the groupSize, except if transformation is in flush state tasks.append( ( ses, taskLfns ) ) return S_OK( tasks )
def _Broadcast( self ): """ This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs. """ if not self.params: return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." ) targetseParam = self.params['TargetSE'] targetSEs = [] sourceSEs = eval( self.params['SourceSE'] ) if targetseParam.count( '[' ): targetSEs = eval( targetseParam ) elif isinstance( targetseParam, list ): targetSEs = targetseParam else: targetSEs = [targetseParam] # sourceSEs = eval(self.params['SourceSE']) # targetSEs = eval(self.params['TargetSE']) destinations = int( self.params.get( 'Destinations', 0 ) ) if destinations and ( destinations >= len( targetSEs ) ): destinations = 0 status = self.params['Status'] groupSize = self.params['GroupSize'] # Number of files per tasks fileGroups = getFileGroups( self.data ) # groups by SE targetSELfns = {} for replicaSE, lfns in fileGroups.items(): ses = replicaSE.split( ',' ) # sourceSites = self._getSitesForSEs(ses) atSource = False for se in ses: if se in sourceSEs: atSource = True if not atSource: continue for lfn in lfns: targets = [] sources = self._getSitesForSEs( ses ) random.shuffle( targetSEs ) for targetSE in targetSEs: site = self._getSiteForSE( targetSE )['Value'] if not site in sources: if ( destinations ) and ( len( targets ) >= destinations ): continue sources.append( site ) targets.append( targetSE ) # after all, if someone wants to copy to the source, it's his choice strTargetSEs = str.join( ',', sorted( targets ) ) if not targetSELfns.has_key( strTargetSEs ): targetSELfns[strTargetSEs] = [] targetSELfns[strTargetSEs].append( lfn ) tasks = [] for ses, lfns in targetSELfns.items(): tasksLfns = breakListIntoChunks( lfns, groupSize ) for taskLfns in tasksLfns: if ( status == 'Flush' ) or ( len( taskLfns ) >= int( groupSize ) ): # do not allow groups smaller than the groupSize, except if transformation is in flush state tasks.append( ( ses, taskLfns ) ) return S_OK( tasks )
def _Broadcast(self): """This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs. Parameters used by this plugin: * SourceSE: Optional: only files at this location are treated * TargetSE: Where to broadcast files to * Destinations: Optional: integer, files are only broadcast to this number of TargetSEs, Destinations has to be larger than the number of TargetSEs * GroupSize: number of files per task """ if not self.params: return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." ) sourceSEs = set( self.util.seParamtoList(self.params.get("SourceSE", []))) targetSEs = self.util.seParamtoList(self.params["TargetSE"]) destinations = int(self.params.get("Destinations", 0)) if destinations and (destinations >= len(targetSEs)): destinations = 0 status = self.params["Status"] groupSize = self.params["GroupSize"] # Number of files per tasks fileGroups = getFileGroups(self.data) # groups by SE targetSELfns = {} for replicaSE, lfns in fileGroups.items(): ses = replicaSE.split(",") atSource = (not sourceSEs) or set(ses).intersection(sourceSEs) if not atSource: continue for lfn in lfns: targets = [] sourceSites = self._getSitesForSEs(ses) random.shuffle(targetSEs) for targetSE in targetSEs: site = self._getSiteForSE(targetSE)["Value"] if site not in sourceSites: if (destinations) and (len(targets) >= destinations): continue sourceSites.append(site) targets.append( targetSE ) # after all, if someone wants to copy to the source, it's his choice strTargetSEs = ",".join(sorted(targets)) targetSELfns.setdefault(strTargetSEs, []).append(lfn) tasks = [] for ses, lfns in targetSELfns.items(): tasksLfns = breakListIntoChunks(lfns, groupSize) for taskLfns in tasksLfns: if (status == "Flush") or (len(taskLfns) >= int(groupSize)): # do not allow groups smaller than the groupSize, except if transformation is in flush state tasks.append((ses, taskLfns)) return S_OK(tasks)