def get_file(self, name):
## Get the file data from PhEDEx
result = self._phedex.make_request('data',
['file=' + name, 'level=file'])
try:
dataset_entry = result[0]['dataset'][0]
block_entry = dataset_entry['block'][0]
file_entry = block_entry['file'][0]
except:
return None
if not self.check_allowed_deataset(dataset_entry['name']):
return None
bname = block_entry['name']
block_name = Block.to_internal_name(bname[bname.find('#') + 1:])
# Just need a named object
dataset = Dataset(dataset_entry['name'])
block = Block(block_name, dataset)
lfile = self._create_file(file_entry, block)
return lfile
def _load_blocks(self, inventory, id_dataset_map, id_block_maps):
sql = 'SELECT b.`id`, b.`dataset_id`, b.`name`, b.`size`, b.`num_files`, b.`is_open`, UNIX_TIMESTAMP(b.`last_update`) FROM `blocks` AS b'
if self._mysql.table_exists('datasets_load_tmp'):
sql += ' INNER JOIN `datasets_load_tmp` AS t ON t.`id` = b.`dataset_id`'
sql += ' ORDER BY b.`dataset_id`'
_dataset_id = 0
dataset = None
for block_id, dataset_id, name, size, num_files, is_open, last_update in self._mysql.xquery(
sql):
if dataset_id != _dataset_id:
_dataset_id = dataset_id
dataset = id_dataset_map[dataset_id]
dataset.blocks.clear()
dataset.size = 0
dataset.num_files = 0
id_block_map = id_block_maps[dataset_id] = {}
block = Block(Block.to_internal_name(name), dataset, size,
num_files, (is_open == 1), last_update)
dataset.blocks.add(block)
dataset.size += block.size
dataset.num_files += block.num_files
id_block_map[block_id] = block
def _create_block(self, block_entry, dataset):
"""
Create a block object with files from a PhEDEx block entry
"""
bname = block_entry['name']
block_name = Block.to_internal_name(bname[bname.find('#') + 1:])
block = Block(block_name,
dataset,
size=block_entry['bytes'],
num_files=block_entry['files'],
is_open=(block_entry['is_open'] == 'y'))
if 'time_update' in block_entry and block_entry[
'time_update'] is not None:
block.last_update = int(block_entry['time_update'])
else:
block.last_update = int(block_entry['time_create'])
return block
def make_block_replicas(dataset_entries, replica_maker):
"""Return a list of block replicas linked to Dataset, Block, Site, and Group"""
block_replicas = []
for dataset_entry in dataset_entries:
dataset = Dataset(dataset_entry['name'])
for block_entry in dataset_entry['block']:
name = block_entry['name']
try:
block_name = Block.to_internal_name(name[name.find('#') +
1:])
except ValueError: # invalid name
continue
block = Block(block_name, dataset, block_entry['bytes'])
block_replicas.extend(replica_maker(block, block_entry))
return block_replicas
def get_file(self, name, block=None):
## Get the file data from PhEDEx
result = self._phedex.make_request('data',
['file=' + name, 'level=file'])
try:
block_entry = result[0]['dataset'][0]['block'][0]
file_entry = block_entry['file'][0]
except:
return None
bname = block_entry['name']
block_name = Block.to_internal_name(bname[bname.find('#') + 1:])
if block is None:
link_block = False
# Just need a named object
dataset = Dataset(dataset_entry['name'])
block = Block(block_name, dataset)
else:
link_block = True
if block.name != block_name:
raise IntegrityError(
'Inconsistent block %s passed to get_file(%s)',
block.full_name(), name)
lfile = self._create_file(file_entry, block)
if link_block:
# Caution - by adding this file we edit the block properties too
existing = block.find_file(lfile.fid())
if existing is None:
block.add_file(lfile)
else:
block.remove_file(existing)
block.add_file(lfile)
return lfile
def get_files(self, dataset_or_block): #override
files = set()
if type(dataset_or_block) is Dataset:
result = self._phedex.make_request(
'data', ['dataset=' + dataset_or_block.name, 'level=file'])
blocks = dict((b.name, b) for b in dataset_or_block.blocks)
else:
result = self._phedex.make_request(
'data',
['block=' + dataset_or_block.full_name(), 'level=file'])
blocks = {dataset_or_block.name: dataset_or_block}
try:
block_entries = result[0]['dataset'][0]['block']
except:
return files
for block_entry in block_entries:
try:
file_entries = block_entry['file']
except:
continue
bname = block_entry['name']
block_name = Block.to_internal_name(bname[bname.find('#') + 1:])
try:
block = blocks[block_name]
except:
# unknown block! maybe should raise?
continue
for file_entry in file_entries:
files.add(self._create_file(file_entry, block))
return files
def update(self, inventory):
for dataset in inventory.datasets.itervalues():
try:
dataset.attr.pop('locked_blocks')
except KeyError:
pass
for source, content_type, site_pattern, lock_url in self._sources.itervalues(
):
if lock_url is not None:
# check that the lock files themselves are not locked
while True:
# Hacky but this is temporary any way
opener = urllib2.build_opener(
webservice.HTTPSCertKeyHandler(Configuration()))
opener.addheaders.append(('Accept', 'application/json'))
request = urllib2.Request(lock_url)
try:
opener.open(request)
except urllib2.HTTPError as err:
if err.code == 404:
# file not found -> no lock
break
else:
raise
LOG.info(
'Lock files are being produced. Waiting 60 seconds.')
time.sleep(60)
LOG.info('Retrieving lock information from %s', source.url_base)
data = source.make_request()
if content_type == WebReplicaLock.LIST_OF_DATASETS:
# simple list of datasets
for dataset_name in data:
if dataset_name is None:
LOG.debug('Dataset name None found in %s',
source.url_base)
continue
try:
dataset = inventory.datasets[dataset_name]
except KeyError:
LOG.debug('Unknown dataset %s in %s', dataset_name,
source.url_base)
continue
if dataset.replicas is None:
continue
try:
locked_blocks = dataset.attr['locked_blocks']
except KeyError:
locked_blocks = dataset.attr['locked_blocks'] = {}
for replica in dataset.replicas:
if site_pattern is not None and not fnmatch.fnmatch(
replica.site.name, site_pattern):
continue
if replica.site in locked_blocks:
locked_blocks[replica.site].update(
brep.block for brep in replica.block_replicas)
else:
locked_blocks[replica.site] = set(
brep.block for brep in replica.block_replicas)
elif content_type == WebReplicaLock.CMSWEB_LIST_OF_DATASETS:
# data['result'] -> simple list of datasets
for dataset_name in data['result']:
if dataset_name is None:
LOG.debug('Dataset name None found in %s',
source.url_base)
continue
try:
dataset = inventory.datasets[dataset_name]
except KeyError:
LOG.debug('Unknown dataset %s in %s', dataset_name,
source.url_base)
continue
if dataset.replicas is None:
continue
try:
locked_blocks = dataset.attr['locked_blocks']
except KeyError:
locked_blocks = dataset.attr['locked_blocks'] = {}
for replica in dataset.replicas:
if site_pattern is not None and not fnmatch.fnmatch(
replica.site.name, site_pattern):
continue
if replica.site in locked_blocks:
locked_blocks[replica.site].update(
brep.block for brep in replica.block_replicas)
else:
locked_blocks[replica.site] = set(
brep.block for brep in replica.block_replicas)
elif content_type == WebReplicaLock.SITE_TO_DATASETS:
# data = {site: {dataset: info}}
for site_name, objects in data.items():
try:
site = inventory.sites[site_name]
except KeyError:
LOG.debug('Unknown site %s in %s', site_name,
source.url_base)
continue
for object_name, info in objects.items():
if not info['lock']:
LOG.debug('Object %s is not locked at %s',
object_name, site_name)
continue
if '#' in object_name:
dataset_name, block_real_name = object_name.split(
'#')
else:
dataset_name = object_name
block_real_name = None
try:
dataset = inventory.datasets[dataset_name]
except KeyError:
LOG.debug('Unknown dataset %s in %s', dataset_name,
source.url_base)
continue
replica = site.find_dataset_replica(dataset)
if replica is None:
LOG.debug('Replica of %s is not at %s in %s',
dataset_name, site_name, source.url_base)
continue
if block_real_name is None:
blocks = list(dataset.blocks)
else:
block = dataset.find_block(
Block.to_internal_name(block_real_name))
if block is None:
LOG.debug('Unknown block %s of %s in %s',
block_real_name, dataset_name,
source.url_base)
continue
blocks = [block]
try:
locked_blocks = dataset.attr['locked_blocks']
except KeyError:
locked_blocks = dataset.attr['locked_blocks'] = {}
if site in locked_blocks:
locked_blocks[site].update(blocks)
else:
locked_blocks[site] = set(blocks)