def _Broadcast( self ):
""" This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs.
"""
if not self.params:
return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." )
targetseParam = self.params['TargetSE']
targetSEs = []
sourceSEs = eval( self.params['SourceSE'] )
if targetseParam.count( '[' ):
targetSEs = eval( targetseParam )
elif isinstance( targetseParam, list ):
targetSEs = targetseParam
else:
targetSEs = [targetseParam]
# sourceSEs = eval(self.params['SourceSE'])
# targetSEs = eval(self.params['TargetSE'])
destinations = int( self.params.get( 'Destinations', 0 ) )
if destinations and ( destinations >= len( targetSEs ) ):
destinations = 0
status = self.params['Status']
groupSize = self.params['GroupSize'] # Number of files per tasks
fileGroups = getFileGroups( self.data ) # groups by SE
targetSELfns = {}
for replicaSE, lfns in fileGroups.items():
ses = replicaSE.split( ',' )
# sourceSites = self._getSitesForSEs(ses)
atSource = False
for se in ses:
if se in sourceSEs:
atSource = True
if not atSource:
continue
for lfn in lfns:
targets = []
sources = self._getSitesForSEs( ses )
random.shuffle( targetSEs )
for targetSE in targetSEs:
site = self._getSiteForSE( targetSE )['Value']
if not site in sources:
if ( destinations ) and ( len( targets ) >= destinations ):
continue
sources.append( site )
targets.append( targetSE ) # after all, if someone wants to copy to the source, it's his choice
strTargetSEs = str.join( ',', sorted( targets ) )
if not targetSELfns.has_key( strTargetSEs ):
targetSELfns[strTargetSEs] = []
targetSELfns[strTargetSEs].append( lfn )
tasks = []
for ses, lfns in targetSELfns.items():
tasksLfns = breakListIntoChunks( lfns, groupSize )
for taskLfns in tasksLfns:
if ( status == 'Flush' ) or ( len( taskLfns ) >= int( groupSize ) ):
# do not allow groups smaller than the groupSize, except if transformation is in flush state
tasks.append( ( ses, taskLfns ) )
return S_OK( tasks )
def _Broadcast( self ):
""" This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs.
"""
if not self.params:
return S_ERROR( "TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters." )
targetseParam = self.params['TargetSE']
targetSEs = []
sourceSEs = eval( self.params['SourceSE'] )
if targetseParam.count( '[' ):
targetSEs = eval( targetseParam )
elif isinstance( targetseParam, list ):
targetSEs = targetseParam
else:
targetSEs = [targetseParam]
# sourceSEs = eval(self.params['SourceSE'])
# targetSEs = eval(self.params['TargetSE'])
destinations = int( self.params.get( 'Destinations', 0 ) )
if destinations and ( destinations >= len( targetSEs ) ):
destinations = 0
status = self.params['Status']
groupSize = self.params['GroupSize'] # Number of files per tasks
fileGroups = getFileGroups( self.data ) # groups by SE
targetSELfns = {}
for replicaSE, lfns in fileGroups.items():
ses = replicaSE.split( ',' )
# sourceSites = self._getSitesForSEs(ses)
atSource = False
for se in ses:
if se in sourceSEs:
atSource = True
if not atSource:
continue
for lfn in lfns:
targets = []
sources = self._getSitesForSEs( ses )
random.shuffle( targetSEs )
for targetSE in targetSEs:
site = self._getSiteForSE( targetSE )['Value']
if not site in sources:
if ( destinations ) and ( len( targets ) >= destinations ):
continue
sources.append( site )
targets.append( targetSE ) # after all, if someone wants to copy to the source, it's his choice
strTargetSEs = str.join( ',', sorted( targets ) )
if not targetSELfns.has_key( strTargetSEs ):
targetSELfns[strTargetSEs] = []
targetSELfns[strTargetSEs].append( lfn )
tasks = []
for ses, lfns in targetSELfns.items():
tasksLfns = breakListIntoChunks( lfns, groupSize )
for taskLfns in tasksLfns:
if ( status == 'Flush' ) or ( len( taskLfns ) >= int( groupSize ) ):
# do not allow groups smaller than the groupSize, except if transformation is in flush state
tasks.append( ( ses, taskLfns ) )
return S_OK( tasks )
def _Broadcast(self):
"""This plug-in takes files found at the sourceSE and broadcasts to all (or a selection of) targetSEs.
Parameters used by this plugin:
* SourceSE: Optional: only files at this location are treated
* TargetSE: Where to broadcast files to
* Destinations: Optional: integer, files are only broadcast to this number of TargetSEs, Destinations has to be
larger than the number of TargetSEs
* GroupSize: number of files per task
"""
if not self.params:
return S_ERROR(
"TransformationPlugin._Broadcast: The 'Broadcast' plugin requires additional parameters."
)
sourceSEs = set(
self.util.seParamtoList(self.params.get("SourceSE", [])))
targetSEs = self.util.seParamtoList(self.params["TargetSE"])
destinations = int(self.params.get("Destinations", 0))
if destinations and (destinations >= len(targetSEs)):
destinations = 0
status = self.params["Status"]
groupSize = self.params["GroupSize"] # Number of files per tasks
fileGroups = getFileGroups(self.data) # groups by SE
targetSELfns = {}
for replicaSE, lfns in fileGroups.items():
ses = replicaSE.split(",")
atSource = (not sourceSEs) or set(ses).intersection(sourceSEs)
if not atSource:
continue
for lfn in lfns:
targets = []
sourceSites = self._getSitesForSEs(ses)
random.shuffle(targetSEs)
for targetSE in targetSEs:
site = self._getSiteForSE(targetSE)["Value"]
if site not in sourceSites:
if (destinations) and (len(targets) >= destinations):
continue
sourceSites.append(site)
targets.append(
targetSE
) # after all, if someone wants to copy to the source, it's his choice
strTargetSEs = ",".join(sorted(targets))
targetSELfns.setdefault(strTargetSEs, []).append(lfn)
tasks = []
for ses, lfns in targetSELfns.items():
tasksLfns = breakListIntoChunks(lfns, groupSize)
for taskLfns in tasksLfns:
if (status == "Flush") or (len(taskLfns) >= int(groupSize)):
# do not allow groups smaller than the groupSize, except if transformation is in flush state
tasks.append((ses, taskLfns))
return S_OK(tasks)