def embed_into(self, inventory, check=False): try: dataset = inventory.datasets[self._dataset.name] except KeyError: raise ObjectError('Unknown dataset %s', self._dataset.name) try: site = inventory.sites[self._site.name] except KeyError: raise ObjectError('Unknown site %s', self._site.name) replica = dataset.find_replica(site) updated = False if replica is None: replica = DatasetReplica(dataset, site) dataset.replicas.add(replica) site.add_dataset_replica(replica, add_block_replicas=False) updated = True elif check and (replica is self or replica == self): # identical object -> return False if check is requested pass else: replica.copy(self) updated = True if check: return replica, updated else: return replica
def find_block_replica(self, block, must_find=False): if type(block).__name__ == 'Block': try: dataset_replica = self._dataset_replicas[block.dataset] except KeyError: if must_find: raise ObjectError('Could not find replica of %s in %s' % (block.dataset.name, self._name)) else: return None else: return dataset_replica.find_block_replica(block, must_find=must_find) else: # lookup by block name - very inefficient operation for dataset_replica in self._dataset_replicas.itervalues(): for block_replica in dataset_replica.block_replicas: if block_replica.block.name == block: return block_replica if must_find: raise ObjectError('Could not find replica of %s in %s' % (block.full_name(), self._name)) else: return None
def embed_into(self, inventory, check=False): try: site = inventory.sites[self._site_name()] except KeyError: raise ObjectError('Unknown site %s', self._site_name()) try: partition = inventory.partitions[self._partition_name()] except KeyError: raise ObjectError('Unknown partition %s', self._partition_name()) updated = False try: site_partition = site.partitions[partition] except KeyError: IntegrityError('SitePartition %s/%s must exist but does not.', site.name, partition.name) else: if check and (site_partition is self or site_partition == self): # identical object -> return False if check is requested pass else: site_partition.copy(self) updated = True if check: return site_partition, updated else: return site_partition
def copy(self, other): if self._dataset.name() != other._dataset.name(): raise ObjectError( 'Cannot copy a replica of %s into a replica of %s', other._dataset.name, self._dataset.name) if self._site.name != other._site.name: raise ObjectError( 'Cannot copy a replica at %s into a replica at %s', other._site.name, self._site.name)
def copy(self, other): if self._block_full_name() != other._block_full_name(): raise ObjectError( 'Cannot copy a replica of %s into a replica of %s' % (other._block_full_name(), self._block_full_name())) if self._site_name() != other._site_name(): raise ObjectError( 'Cannot copy a replica at %s into a replica at %s' % (other._site.name, self._site_name())) self._copy_no_check(other)
def copy(self, other): if self._site_name() != other._site_name(): raise ObjectError( 'Cannot copy a partition at %s into a partition at %s', other._site_name(), self._site_name()) if self._partition_name() != other._partition_name(): raise ObjectError( 'Cannot copy a site partition of %s into a site partition of %s', other._partition_name(), self._partition_name()) self._quota = other._quota
def copy(self, other): if self._dataset_name() != other._dataset_name(): raise ObjectError( 'Cannot copy a replica of %s into a replica of %s', other._dataset_name(), self._dataset_name()) if self._site_name() != other._site_name(): raise ObjectError( 'Cannot copy a replica at %s into a replica at %s', other._site_name(), self._site_name()) self.growing = other.growing self.group = other.group
def embed_into(self, inventory, check=False): try: dataset = inventory.datasets[self._dataset_name()] except KeyError: raise ObjectError('Unknown dataset %s' % (self._dataset_name())) block = dataset.find_block(self._block_name(), must_find=True) try: site = inventory.sites[self._site_name()] except KeyError: raise ObjectError('Unknown site %s' % (self._site_name())) try: group = inventory.groups[self._group_name()] except KeyError: raise ObjectError('Unknown group %s' % (self._group_name())) replica = block.find_replica(site) updated = False if replica is None: replica = BlockReplica(block, site, group, self.is_custodial, self.size, self.last_update, self.file_ids) dataset_replica = site.find_dataset_replica(dataset, must_find=True) dataset_replica.block_replicas.add(replica) block.replicas.add(replica) site.add_block_replica(replica) updated = True elif check and (replica is self or replica == self): # identical object -> return False if check is requested pass else: replica.copy(self) if type(self.group) is str or self.group is None: # can happen if self is an unlinked clone replica.group = group if self.size < 0: # self represents a full block replica without the knowledge of the actual size (again an unlinked clone) replica.size = block.size site.update_partitioning(replica) updated = True if check: return replica, updated else: return replica
def copy(self, other): if self._block.full_name() != other._block.full_name(): raise ObjectError( 'Cannot copy a replica of %s into a replica of %s', other._block.full_name(), self._block.full_name()) if self._site.name != other._site.name: raise ObjectError( 'Cannot copy a replica at %s into a replica at %s', other._site.name, self._site.name) self.group = other.group self.is_complete = other.is_complete self.is_custodial = other.is_custodial self.size = other.size self.last_update = other.last_update
def __init__(self, name, dataset, size=0, num_files=0, is_open=False, last_update=0, bid=0, internal_name=True): if internal_name: self._name = name else: if Block.name_pattern is not None and not Block.name_pattern.match( name): raise ObjectError('Invalid block name %s' % name) self._name = Block.to_internal_name(name) self._dataset = dataset self._size = size self._num_files = num_files self.is_open = is_open self.last_update = last_update self.id = bid self.replicas = set() self._files = None
def add_block_replica(self, replica): # this function should be called automatically to avoid integrity errors try: dataset_replica = self._dataset_replicas[replica.block.dataset] except KeyError: raise ObjectError('Dataset %s is not at %s' % (replica.block.dataset.name, self._name)) if replica not in dataset_replica.block_replicas: raise IntegrityError('%s is not a block replica of %s' % (str(replica), str(dataset_replica))) for partition, site_partition in self.partitions.iteritems(): if not partition.contains(replica): continue try: block_replica_list = site_partition.replicas[dataset_replica] except KeyError: if len(dataset_replica.block_replicas) == 1: # this is the sole block replica site_partition.replicas[dataset_replica] = None else: site_partition.replicas[dataset_replica] = set([replica]) else: if block_replica_list is None: # assume this function was called for all new block replicas # then we are just adding another replica to this partition pass else: # again assuming this function is called for all new block replicas, # block_replica_list not being None implies that adding this new # replica will not make the dataset replica in this partition complete block_replica_list.add(replica)
def add_file(self, lfile): if lfile.block != self.block: raise ObjectError('Cannot add file %s (block %s) to %s', lfile.lfn, lfile.block.full_name(), str(self)) if BlockReplica._use_file_ids: if self.file_ids is None: # This was a full replica. A new file was added to the block. The replica remains full. return else: file_ids = set(self.file_ids) if lfile.id == 0: if lfile.lfn in file_ids: return if lfile.lfn in set(f.lfn for f in self.files()): return file_ids.add(lfile.lfn) else: if lfile.id in file_ids: return file_ids.add(lfile.id) self.size += lfile.size if self.size == self.block.size and len( file_ids) == self.block.num_files: self.file_ids = None else: self.file_ids = tuple(file_ids) else: self.file_ids += 1
def embed_into(self, inventory, check=False): try: dataset = inventory.datasets[self._dataset_name()] except KeyError: raise ObjectError('Unknown dataset %s' % self._dataset_name()) block = dataset.find_block(self._name) updated = False if block is None: block = Block(self._name, dataset, self._size, self._num_files, self.is_open, self.last_update, self.id) dataset.blocks.add(block) updated = True elif check and (block is self or block == self): # identical object -> return False if check is requested pass else: # server-side inventory should not load files and just copy the values server_side = hasattr(inventory, 'has_store') block._copy_no_check(self, load_files=(not server_side)) updated = True if check: return block, updated else: return block
def find_block(self, block_name, must_find = False): try: return next(b for b in self.blocks if b.name == block_name) except StopIteration: if must_find: raise ObjectError('Could not find block %s in %s', block_name, self._name) else: return None
def Block_from_full_name(full_name): # return dataset name, block internal name delim = full_name.find('#') if delim == -1: raise ObjectError('Invalid block name %s' % full_name) return full_name[:delim], Block_to_internal_name(full_name[delim + 1:])
def copy(self, other): if self._dataset.name != other.dataset.name: raise ObjectError('Cannot copy a block of %s into a block of %s', other.dataset.name, self._dataset.name) self.size = other.size self.num_files = other.num_files self.is_open = other.is_open self.last_update = other.last_update
def find_dataset_replica(self, dataset, must_find=False): try: return self._dataset_replicas[dataset] except KeyError: if must_find: raise ObjectError('Could not find replica of %s in %s' % (dataset.name, self._name)) else: return None
def embed_into(self, inventory, check=False): try: dataset = inventory.datasets[self._dataset_name()] except KeyError: raise ObjectError('Unknown dataset %s', self._dataset_name()) try: site = inventory.sites[self._site_name()] except KeyError: raise ObjectError('Unknown site %s', self._site_name()) if self._group_name() is not None: try: group = inventory.groups[self._group_name()] except KeyError: raise ObjectError('Unknown group %s' % (self._group_name())) else: group = None replica = dataset.find_replica(site) updated = False if replica is None: replica = DatasetReplica(dataset, site, self.growing, group) dataset.replicas.add(replica) site.add_dataset_replica(replica, add_block_replicas=False) updated = True elif check and (replica is self or replica == self): # identical object -> return False if check is requested pass else: replica.copy(self) if type(self.group) is str or self.group is None: # can happen if self is an unlinked clone replica.group = group updated = True if check: return replica, updated else: return replica
def Block_from_full_name(full_name): """ @param full_name Full name of the block @return (dataset name, block internal name) """ delim = full_name.find('#') if delim == -1: raise ObjectError('Invalid block name %s' % full_name) return full_name[:delim], Block_to_internal_name(full_name[delim + 1:])
def embed_into(self, inventory, check=False): try: dataset = inventory.datasets[self._block.dataset.name] except KeyError: raise ObjectError('Unknown dataset %s', self._block.dataset.name) block = dataset.find_block(self._block.name, must_find=True) try: site = inventory.sites[self._site.name] except KeyError: raise ObjectError('Unknown site %s', self._site.name) try: group = inventory.groups[self.group.name] except KeyError: raise ObjectError('Unknown group %s', self.group.name) replica = block.find_replica(site) updated = False if replica is None: replica = BlockReplica(block, site, group, self.is_complete, self.is_custodial, self.size, self.last_update) dataset_replica = dataset.find_replica(site, must_find=True) dataset_replica.block_replicas.add(replica) block.replicas.add(replica) site.add_block_replica(replica) updated = True elif check and (replica is self or replica == self): # identical object -> return False if check is requested pass else: replica.copy(self) site.update_partitioning(replica) updated = True if check: return replica, updated else: return replica
def find_file(self, path, must_find=False): for block in self.blocks: f = block.find_file(path) if f is not None: return f if must_find: raise ObjectError('Could not find file %s in %s', path, self._name) else: return None
def find_block(self, block): if self.blocks is None: raise ObjectError('Blocks are not loaded for %s' % self.name) try: if type(block).__name__ == 'Block': return next(b for b in self.blocks if b == block) else: return next(b for b in self.blocks if b.name == block) except StopIteration: return None
def write_into(self, store): if BlockReplica._use_file_ids and self.file_ids is not None: for fid in self.file_ids: try: fid += 0 except TypeError: # was some string raise ObjectError( 'Cannot write %s into store because one of the files %s %s is not known yet' % (str(self), fid, type(fid).__name__)) store.save_blockreplica(self)
def find_replica(self, site, must_find = False): try: if type(site) is str: return next(r for r in self.replicas if r.site.name == site) else: return next(r for r in self.replicas if r.site == site) except StopIteration: if must_find: raise ObjectError('Could not find replica on %s of %s', str(site), self._name) else: return None
def delete_from(self, inventory): if self._name is None: raise ObjectError('Deletion of null group not allowed') # Pop the group from the main list. All block replicas owned by the group # will be disowned. group = inventory.groups.pop(self._name) for dataset in inventory.datasets.itervalues(): for replica in dataset.replicas: for block_replica in replica.block_replicas: if block_replica.group == group: block_replica.group = inventory.groups[None]
def Dataset_format_software_version(value): if type(value) is str: formatted = eval(value) elif type(value) is not tuple: # some iterable formatted = tuple(value) else: formatted = value if type(formatted) is not tuple or len(formatted) != 2: raise ObjectError('Invalid software version %s' % repr(value)) return formatted
def find_file(self, lfn, must_find=False): """ @param lfn File name @param must_find Raise an exception if file is not found. """ try: return next(f for f in self.files if f._lfn == lfn) except StopIteration: if must_find: raise ObjectError('Cannot find file %s' % str(lfn)) else: return None
def find_replica(self, site, must_find=False): try: if type(site) is str: return next(r for r in self.replicas if r.site.name == site) else: return next(r for r in self.replicas if r.site == site) except StopIteration: if must_find: raise ObjectError('Cannot find replica at %s for %s' % (site.name, self.full_name())) else: return None
def find_block_replica(self, block, must_find=False): try: if type(block).__name__ == 'Block': return next(b for b in self.block_replicas if b.block == block) else: return next(b for b in self.block_replicas if b.block.name == block) except StopIteration: if must_find: raise ObjectError('Cannot find block replica %s/%s', self._site.name, block.full_name()) else: return None
def update_file(self, path, size): if self.files is None: raise ObjectError('Files are not loaded for %s' % self.name) directory_id = File.get_directory_id(path) name = File.get_basename(path) old_file = next(f for f in self.files if f.name == name and f.directory_id == directory_id) self.files.remove(old_file) new_file = old_file.clone(size=size) self.files.add(new_file) return new_file