def embed_into(self, inventory, check=False):
try:
dataset = inventory.datasets[self._dataset.name]
except KeyError:
raise ObjectError('Unknown dataset %s', self._dataset.name)
try:
site = inventory.sites[self._site.name]
except KeyError:
raise ObjectError('Unknown site %s', self._site.name)
replica = dataset.find_replica(site)
updated = False
if replica is None:
replica = DatasetReplica(dataset, site)
dataset.replicas.add(replica)
site.add_dataset_replica(replica, add_block_replicas=False)
updated = True
elif check and (replica is self or replica == self):
# identical object -> return False if check is requested
pass
else:
replica.copy(self)
updated = True
if check:
return replica, updated
else:
return replica
def find_block_replica(self, block, must_find=False):
if type(block).__name__ == 'Block':
try:
dataset_replica = self._dataset_replicas[block.dataset]
except KeyError:
if must_find:
raise ObjectError('Could not find replica of %s in %s' %
(block.dataset.name, self._name))
else:
return None
else:
return dataset_replica.find_block_replica(block,
must_find=must_find)
else:
# lookup by block name - very inefficient operation
for dataset_replica in self._dataset_replicas.itervalues():
for block_replica in dataset_replica.block_replicas:
if block_replica.block.name == block:
return block_replica
if must_find:
raise ObjectError('Could not find replica of %s in %s' %
(block.full_name(), self._name))
else:
return None
def embed_into(self, inventory, check=False):
try:
site = inventory.sites[self._site_name()]
except KeyError:
raise ObjectError('Unknown site %s', self._site_name())
try:
partition = inventory.partitions[self._partition_name()]
except KeyError:
raise ObjectError('Unknown partition %s', self._partition_name())
updated = False
try:
site_partition = site.partitions[partition]
except KeyError:
IntegrityError('SitePartition %s/%s must exist but does not.',
site.name, partition.name)
else:
if check and (site_partition is self or site_partition == self):
# identical object -> return False if check is requested
pass
else:
site_partition.copy(self)
updated = True
if check:
return site_partition, updated
else:
return site_partition
def copy(self, other):
if self._dataset.name() != other._dataset.name():
raise ObjectError(
'Cannot copy a replica of %s into a replica of %s',
other._dataset.name, self._dataset.name)
if self._site.name != other._site.name:
raise ObjectError(
'Cannot copy a replica at %s into a replica at %s',
other._site.name, self._site.name)
def copy(self, other):
if self._block_full_name() != other._block_full_name():
raise ObjectError(
'Cannot copy a replica of %s into a replica of %s' %
(other._block_full_name(), self._block_full_name()))
if self._site_name() != other._site_name():
raise ObjectError(
'Cannot copy a replica at %s into a replica at %s' %
(other._site.name, self._site_name()))
self._copy_no_check(other)
def copy(self, other):
if self._site_name() != other._site_name():
raise ObjectError(
'Cannot copy a partition at %s into a partition at %s',
other._site_name(), self._site_name())
if self._partition_name() != other._partition_name():
raise ObjectError(
'Cannot copy a site partition of %s into a site partition of %s',
other._partition_name(), self._partition_name())
self._quota = other._quota
def copy(self, other):
if self._dataset_name() != other._dataset_name():
raise ObjectError(
'Cannot copy a replica of %s into a replica of %s',
other._dataset_name(), self._dataset_name())
if self._site_name() != other._site_name():
raise ObjectError(
'Cannot copy a replica at %s into a replica at %s',
other._site_name(), self._site_name())
self.growing = other.growing
self.group = other.group
def embed_into(self, inventory, check=False):
try:
dataset = inventory.datasets[self._dataset_name()]
except KeyError:
raise ObjectError('Unknown dataset %s' % (self._dataset_name()))
block = dataset.find_block(self._block_name(), must_find=True)
try:
site = inventory.sites[self._site_name()]
except KeyError:
raise ObjectError('Unknown site %s' % (self._site_name()))
try:
group = inventory.groups[self._group_name()]
except KeyError:
raise ObjectError('Unknown group %s' % (self._group_name()))
replica = block.find_replica(site)
updated = False
if replica is None:
replica = BlockReplica(block, site, group, self.is_custodial,
self.size, self.last_update, self.file_ids)
dataset_replica = site.find_dataset_replica(dataset,
must_find=True)
dataset_replica.block_replicas.add(replica)
block.replicas.add(replica)
site.add_block_replica(replica)
updated = True
elif check and (replica is self or replica == self):
# identical object -> return False if check is requested
pass
else:
replica.copy(self)
if type(self.group) is str or self.group is None:
# can happen if self is an unlinked clone
replica.group = group
if self.size < 0:
# self represents a full block replica without the knowledge of the actual size (again an unlinked clone)
replica.size = block.size
site.update_partitioning(replica)
updated = True
if check:
return replica, updated
else:
return replica
def copy(self, other):
if self._block.full_name() != other._block.full_name():
raise ObjectError(
'Cannot copy a replica of %s into a replica of %s',
other._block.full_name(), self._block.full_name())
if self._site.name != other._site.name:
raise ObjectError(
'Cannot copy a replica at %s into a replica at %s',
other._site.name, self._site.name)
self.group = other.group
self.is_complete = other.is_complete
self.is_custodial = other.is_custodial
self.size = other.size
self.last_update = other.last_update
def __init__(self,
name,
dataset,
size=0,
num_files=0,
is_open=False,
last_update=0,
bid=0,
internal_name=True):
if internal_name:
self._name = name
else:
if Block.name_pattern is not None and not Block.name_pattern.match(
name):
raise ObjectError('Invalid block name %s' % name)
self._name = Block.to_internal_name(name)
self._dataset = dataset
self._size = size
self._num_files = num_files
self.is_open = is_open
self.last_update = last_update
self.id = bid
self.replicas = set()
self._files = None
def add_block_replica(self, replica):
# this function should be called automatically to avoid integrity errors
try:
dataset_replica = self._dataset_replicas[replica.block.dataset]
except KeyError:
raise ObjectError('Dataset %s is not at %s' %
(replica.block.dataset.name, self._name))
if replica not in dataset_replica.block_replicas:
raise IntegrityError('%s is not a block replica of %s' %
(str(replica), str(dataset_replica)))
for partition, site_partition in self.partitions.iteritems():
if not partition.contains(replica):
continue
try:
block_replica_list = site_partition.replicas[dataset_replica]
except KeyError:
if len(dataset_replica.block_replicas) == 1:
# this is the sole block replica
site_partition.replicas[dataset_replica] = None
else:
site_partition.replicas[dataset_replica] = set([replica])
else:
if block_replica_list is None:
# assume this function was called for all new block replicas
# then we are just adding another replica to this partition
pass
else:
# again assuming this function is called for all new block replicas,
# block_replica_list not being None implies that adding this new
# replica will not make the dataset replica in this partition complete
block_replica_list.add(replica)
def add_file(self, lfile):
if lfile.block != self.block:
raise ObjectError('Cannot add file %s (block %s) to %s', lfile.lfn,
lfile.block.full_name(), str(self))
if BlockReplica._use_file_ids:
if self.file_ids is None:
# This was a full replica. A new file was added to the block. The replica remains full.
return
else:
file_ids = set(self.file_ids)
if lfile.id == 0:
if lfile.lfn in file_ids:
return
if lfile.lfn in set(f.lfn for f in self.files()):
return
file_ids.add(lfile.lfn)
else:
if lfile.id in file_ids:
return
file_ids.add(lfile.id)
self.size += lfile.size
if self.size == self.block.size and len(
file_ids) == self.block.num_files:
self.file_ids = None
else:
self.file_ids = tuple(file_ids)
else:
self.file_ids += 1
def embed_into(self, inventory, check=False):
try:
dataset = inventory.datasets[self._dataset_name()]
except KeyError:
raise ObjectError('Unknown dataset %s' % self._dataset_name())
block = dataset.find_block(self._name)
updated = False
if block is None:
block = Block(self._name, dataset, self._size, self._num_files,
self.is_open, self.last_update, self.id)
dataset.blocks.add(block)
updated = True
elif check and (block is self or block == self):
# identical object -> return False if check is requested
pass
else:
# server-side inventory should not load files and just copy the values
server_side = hasattr(inventory, 'has_store')
block._copy_no_check(self, load_files=(not server_side))
updated = True
if check:
return block, updated
else:
return block
def find_block(self, block_name, must_find = False):
try:
return next(b for b in self.blocks if b.name == block_name)
except StopIteration:
if must_find:
raise ObjectError('Could not find block %s in %s', block_name, self._name)
else:
return None
def Block_from_full_name(full_name):
# return dataset name, block internal name
delim = full_name.find('#')
if delim == -1:
raise ObjectError('Invalid block name %s' % full_name)
return full_name[:delim], Block_to_internal_name(full_name[delim + 1:])
def copy(self, other):
if self._dataset.name != other.dataset.name:
raise ObjectError('Cannot copy a block of %s into a block of %s',
other.dataset.name, self._dataset.name)
self.size = other.size
self.num_files = other.num_files
self.is_open = other.is_open
self.last_update = other.last_update
def find_dataset_replica(self, dataset, must_find=False):
try:
return self._dataset_replicas[dataset]
except KeyError:
if must_find:
raise ObjectError('Could not find replica of %s in %s' %
(dataset.name, self._name))
else:
return None
def embed_into(self, inventory, check=False):
try:
dataset = inventory.datasets[self._dataset_name()]
except KeyError:
raise ObjectError('Unknown dataset %s', self._dataset_name())
try:
site = inventory.sites[self._site_name()]
except KeyError:
raise ObjectError('Unknown site %s', self._site_name())
if self._group_name() is not None:
try:
group = inventory.groups[self._group_name()]
except KeyError:
raise ObjectError('Unknown group %s' % (self._group_name()))
else:
group = None
replica = dataset.find_replica(site)
updated = False
if replica is None:
replica = DatasetReplica(dataset, site, self.growing, group)
dataset.replicas.add(replica)
site.add_dataset_replica(replica, add_block_replicas=False)
updated = True
elif check and (replica is self or replica == self):
# identical object -> return False if check is requested
pass
else:
replica.copy(self)
if type(self.group) is str or self.group is None:
# can happen if self is an unlinked clone
replica.group = group
updated = True
if check:
return replica, updated
else:
return replica
def Block_from_full_name(full_name):
"""
@param full_name Full name of the block
@return (dataset name, block internal name)
"""
delim = full_name.find('#')
if delim == -1:
raise ObjectError('Invalid block name %s' % full_name)
return full_name[:delim], Block_to_internal_name(full_name[delim + 1:])
def embed_into(self, inventory, check=False):
try:
dataset = inventory.datasets[self._block.dataset.name]
except KeyError:
raise ObjectError('Unknown dataset %s', self._block.dataset.name)
block = dataset.find_block(self._block.name, must_find=True)
try:
site = inventory.sites[self._site.name]
except KeyError:
raise ObjectError('Unknown site %s', self._site.name)
try:
group = inventory.groups[self.group.name]
except KeyError:
raise ObjectError('Unknown group %s', self.group.name)
replica = block.find_replica(site)
updated = False
if replica is None:
replica = BlockReplica(block, site, group, self.is_complete,
self.is_custodial, self.size,
self.last_update)
dataset_replica = dataset.find_replica(site, must_find=True)
dataset_replica.block_replicas.add(replica)
block.replicas.add(replica)
site.add_block_replica(replica)
updated = True
elif check and (replica is self or replica == self):
# identical object -> return False if check is requested
pass
else:
replica.copy(self)
site.update_partitioning(replica)
updated = True
if check:
return replica, updated
else:
return replica
def find_file(self, path, must_find=False):
for block in self.blocks:
f = block.find_file(path)
if f is not None:
return f
if must_find:
raise ObjectError('Could not find file %s in %s', path, self._name)
else:
return None
def find_block(self, block):
if self.blocks is None:
raise ObjectError('Blocks are not loaded for %s' % self.name)
try:
if type(block).__name__ == 'Block':
return next(b for b in self.blocks if b == block)
else:
return next(b for b in self.blocks if b.name == block)
except StopIteration:
return None
def write_into(self, store):
if BlockReplica._use_file_ids and self.file_ids is not None:
for fid in self.file_ids:
try:
fid += 0
except TypeError:
# was some string
raise ObjectError(
'Cannot write %s into store because one of the files %s %s is not known yet'
% (str(self), fid, type(fid).__name__))
store.save_blockreplica(self)
def find_replica(self, site, must_find = False):
try:
if type(site) is str:
return next(r for r in self.replicas if r.site.name == site)
else:
return next(r for r in self.replicas if r.site == site)
except StopIteration:
if must_find:
raise ObjectError('Could not find replica on %s of %s', str(site), self._name)
else:
return None
def delete_from(self, inventory):
if self._name is None:
raise ObjectError('Deletion of null group not allowed')
# Pop the group from the main list. All block replicas owned by the group
# will be disowned.
group = inventory.groups.pop(self._name)
for dataset in inventory.datasets.itervalues():
for replica in dataset.replicas:
for block_replica in replica.block_replicas:
if block_replica.group == group:
block_replica.group = inventory.groups[None]
def Dataset_format_software_version(value):
if type(value) is str:
formatted = eval(value)
elif type(value) is not tuple:
# some iterable
formatted = tuple(value)
else:
formatted = value
if type(formatted) is not tuple or len(formatted) != 2:
raise ObjectError('Invalid software version %s' % repr(value))
return formatted
def find_file(self, lfn, must_find=False):
"""
@param lfn File name
@param must_find Raise an exception if file is not found.
"""
try:
return next(f for f in self.files if f._lfn == lfn)
except StopIteration:
if must_find:
raise ObjectError('Cannot find file %s' % str(lfn))
else:
return None
def find_replica(self, site, must_find=False):
try:
if type(site) is str:
return next(r for r in self.replicas if r.site.name == site)
else:
return next(r for r in self.replicas if r.site == site)
except StopIteration:
if must_find:
raise ObjectError('Cannot find replica at %s for %s' %
(site.name, self.full_name()))
else:
return None
def find_block_replica(self, block, must_find=False):
try:
if type(block).__name__ == 'Block':
return next(b for b in self.block_replicas if b.block == block)
else:
return next(b for b in self.block_replicas
if b.block.name == block)
except StopIteration:
if must_find:
raise ObjectError('Cannot find block replica %s/%s',
self._site.name, block.full_name())
else:
return None
def update_file(self, path, size):
if self.files is None:
raise ObjectError('Files are not loaded for %s' % self.name)
directory_id = File.get_directory_id(path)
name = File.get_basename(path)
old_file = next(f for f in self.files
if f.name == name and f.directory_id == directory_id)
self.files.remove(old_file)
new_file = old_file.clone(size=size)
self.files.add(new_file)
return new_file