def get_replicas(self, site=None, dataset=None, block=None): #override
if site is None:
site_check = self.check_allowed_site
else:
site_check = None
if not self.check_allowed_site(site):
return []
if dataset is None and block is None:
dataset_check = self.check_allowed_dataset
else:
dataset_check = None
if dataset is not None:
if not self.check_allowed_dataset(dataset):
return []
if block is not None:
if not self.check_allowed_dataset(block[:block.find('#')]):
return []
options = []
if site is not None:
options.append('node=' + site)
if dataset is not None:
options.append('dataset=' + dataset)
if block is not None:
options.append('block=' + block)
LOG.info('get_replicas(' + ','.join(options) +
') Fetching the list of replicas from PhEDEx')
if len(options) == 0:
return []
block_entries = self._phedex.make_request('blockreplicas',
options,
timeout=7200)
parallelizer = Map()
parallelizer.timeout = 7200
# Automatically starts a thread as we add the output of block_entries
combine_file = parallelizer.get_starter(self._combine_file_info)
for block_entry in block_entries:
for replica_entry in block_entry['replica']:
if replica_entry['complete'] == 'n':
break
else:
continue
# there is at least one incomplete replica
try:
dataset_name, block_name = Block.from_full_name(
block_entry['name'])
except ObjectError: # invalid name
continue
if dataset_check and not dataset_check(dataset_name):
continue
combine_file.add_input(block_entry)
combine_file.close()
# _combine_file_info alters block_entries directly - no need to deal with output
combine_file.get_outputs()
block_replicas = PhEDExReplicaInfoSource.make_block_replicas(
block_entries,
PhEDExReplicaInfoSource.maker_blockreplicas,
site_check=site_check,
dataset_check=dataset_check)
# Also use subscriptions call which has a lower latency than blockreplicas
# For example, group change on a block replica at time T may not show up in blockreplicas until up to T + 15 minutes
# while in subscriptions it is visible within a few seconds
# But subscriptions call without a dataset or block takes too long
if dataset is None and block is None:
return block_replicas
indexed = collections.defaultdict(dict)
for replica in block_replicas:
indexed[(replica.site.name,
replica.block.dataset.name)][replica.block.name] = replica
dataset_entries = self._phedex.make_request('subscriptions',
options,
timeout=3600)
for dataset_entry in dataset_entries:
dataset_name = dataset_entry['name']
if not self.check_allowed_dataset(dataset_name):
continue
try:
subscriptions = dataset_entry['subscription']
except KeyError:
pass
else:
for sub_entry in subscriptions:
site_name = sub_entry['node']
if not self.check_allowed_site(site_name):
continue
replicas = indexed[(site_name, dataset_name)]
for replica in replicas.itervalues():
replica.group = Group(sub_entry['group'])
replica.is_custodial = (sub_entry['custodial'] == 'y')
try:
block_entries = dataset_entry['block']
except KeyError:
pass
else:
for block_entry in block_entries:
try:
_, block_name = Block.from_full_name(
block_entry['name'])
except ObjectError:
continue
try:
subscriptions = block_entry['subscription']
except KeyError:
continue
for sub_entry in subscriptions:
site_name = sub_entry['node']
if not self.check_allowed_site(site_name):
continue
try:
replica = indexed[(site_name,
dataset_name)][block_name]
except KeyError:
continue
replica.group = Group(sub_entry['group'])
if sub_entry['node_bytes'] == block_entry['bytes']:
# complete
replica.size = sub_entry['node_bytes']
if replica.size is None:
replica.size = 0
replica.files = None
else:
# incomplete - since we cannot know what files are there, we'll just have to pretend there is none
replica.size = 0
replica.files = tuple()
replica.is_custodial = (sub_entry['custodial'] == 'y')
if sub_entry['time_update'] is not None:
replica.last_update = 0
else:
replica.last_update = int(sub_entry['time_update'])
return block_replicas
def get_updated_replicas(self, updated_since, inventory): #override
LOG.info(
'get_updated_replicas(%d) Fetching the list of replicas from PhEDEx',
updated_since)
nodes = []
for entry in self._phedex.make_request('nodes', timeout=600):
if not self.check_allowed_site(entry['name']):
continue
if entry['name'] not in inventory.sites:
continue
nodes.append(entry['name'])
try:
tmpconfig = Configuration(
self._parallelizer_config.get('parallel', None))
except Exception as e:
LOG.error(str(e))
tmpconfig = Configuration()
parallelizer = Map(tmpconfig)
parallelizer.timeout = 5400
def get_node_replicas(node):
options = ['update_since=%d' % updated_since, 'node=%s' % node]
results = self._phedex.make_request('blockreplicas', options)
return node, results
# Use async to fire threads on demand
node_results = parallelizer.execute(get_node_replicas,
nodes,
async=True)
# Automatically starts a thread as we add the output of block_replicas
combine_file = parallelizer.get_starter(self._combine_file_info)
all_block_entries = []
for node, block_entries in node_results:
site = inventory.sites[node]
for block_entry in block_entries:
all_block_entries.append(block_entry)
replica_entry = block_entry['replica'][0]
if replica_entry['complete'] == 'y':
continue
# incomplete block replica - should we fetch file info?
try:
dataset_name, block_name = Block.from_full_name(
block_entry['name'])
except ObjectError:
pass
else:
try:
dataset = inventory.datasets[dataset_name]
block = dataset.find_block(block_name)
replica = block.find_replica(site)
if replica.file_ids is None:
num_files = block.num_files
else:
num_files = len(replica.file_ids)
if replica.size == replica_entry[
'bytes'] and num_files == replica_entry[
'files']:
# no we don't have to
continue
except:
# At any point of the above lookups we may hit a None object or KeyError or what not
pass
LOG.debug(
'Replica %s:%s is incomplete. Fetching file information.',
replica_entry['node'], block_entry['name'])
combine_file.add_input(block_entry)
combine_file.close()
# _combine_file_info alters block_entries directly - no need to deal with output
combine_file.get_outputs()
LOG.info('get_updated_replicas(%d) Got outputs' % updated_since)
return PhEDExReplicaInfoSource.make_block_replicas(
all_block_entries,
PhEDExReplicaInfoSource.maker_blockreplicas,
dataset_check=self.check_allowed_dataset)