def schedule_copies(self, replica_list, operation_id, comments = ''): #override sites = set(r.site for r in replica_list) if len(sites) != 1: raise OperationalError('schedule_copies should be called with a list of replicas at a single site.') LOG.info('Scheduling copy of %d replicas to %s using RLFSM (operation %d)', len(replica_list), list(sites)[0], operation_id) result = [] for replica in replica_list: # Function spec is to return clones (so that if specific block fails to copy, we can return a dataset replica without the block) clone_replica = DatasetReplica(replica.dataset, replica.site) clone_replica.copy(replica) result.append(clone_replica) for block_replica in replica.block_replicas: LOG.debug('Subscribing files for %s', str(block_replica)) if block_replica.file_ids is None: LOG.debug('No file to subscribe for %s', str(block_replica)) return all_files = block_replica.block.files missing_files = all_files - block_replica.files() for lfile in missing_files: self.rlfsm.subscribe_file(block_replica.site, lfile) clone_block_replica = BlockReplica(block_replica.block, block_replica.site, block_replica.group) clone_block_replica.copy(block_replica) clone_block_replica.last_update = int(time.time()) clone_replica.block_replicas.add(clone_block_replica) # no external dependency - everything is a success return result
def schedule_deletions(self, replica_list, operation_id, comments=''): #override LOG.info('Ignoring deletion schedule of %d replicas (operation %d)', len(replica_list), operation_id) result = [] for replica, block_replicas in replica_list: clone_replica = DatasetReplica(replica.dataset, replica.site) clone_replica.copy(replica) if block_replicas is None: result.append((clone_replica, None)) else: clone_block_replicas = [] for block_replica in block_replicas: clone_block_replica = BlockReplica(block_replica.block, block_replica.site, block_replica.group) clone_block_replica.copy(block_replica) clone_block_replica.last_update = int(time.time()) clone_block_replicas.append(clone_block_replica) result.append((clone_replica, clone_block_replicas)) return result
def schedule_deletions(self, replica_list, operation_id, comments=''): #override sites = set(r.site for r, b in replica_list) if len(sites) != 1: raise OperationalError( 'schedule_copies should be called with a list of replicas at a single site.' ) site = list(sites)[0] LOG.info( 'Scheduling deletion of %d replicas from %s using RLFSM (operation %d)', len(replica_list), site.name, operation_id) clones = [] for dataset_replica, block_replicas in replica_list: if block_replicas is None: to_delete = dataset_replica.block_replicas else: to_delete = block_replicas for block_replica in to_delete: for lfile in block_replica.files(): self.rlfsm.desubscribe_file(block_replica.site, lfile) # No external dependency -> all operations are successful clone_replica = DatasetReplica(dataset_replica.dataset, dataset_replica.site) clone_replica.copy(dataset_replica) if block_replicas is None: clones.append((clone_replica, None)) else: clones.append((clone_replica, [])) for block_replica in block_replicas: clone_block_replica = BlockReplica(block_replica.block, block_replica.site) clone_block_replica.copy(block_replica) clone_block_replica.last_update = int(time.time()) clones[-1][1].append(clone_block_replica) return clones
def schedule_copies(self, replica_list, operation_id, comments = ''): #override LOG.info('Ignoring copy schedule of %d replicas (operation %d)', len(replica_list), operation_id) result = [] for replica in replica_list: clone_replica = DatasetReplica(replica.dataset, replica.site) clone_replica.copy(replica) result.append(clone_replica) for block_replica in replica.block_replicas: clone_block_replica = BlockReplica(block_replica.block, block_replica.site, block_replica.group) clone_block_replica.copy(block_replica) clone_block_replica.last_update = int(time.time()) clone_replica.block_replicas.add(clone_block_replica) return result
def schedule_copies(self, replica_list, operation_id, comments = ''): #override sites = set(r.site for r in replica_list) if len(sites) != 1: raise OperationalError('schedule_copies should be called with a list of replicas at a single site.') LOG.info('Scheduling copy of %d replicas to %s using RLFSM (operation %d)', len(replica_list), list(sites)[0], operation_id) result = [] for replica in replica_list: # Function spec is to return clones (so that if specific block fails to copy, we can return a dataset replica without the block) clone_replica = DatasetReplica(replica.dataset, replica.site) clone_replica.copy(replica) result.append(clone_replica) for block_replica in replica.block_replicas: LOG.debug('Subscribing files for %s', str(block_replica)) if block_replica.file_ids is None: LOG.debug('No file to subscribe for %s', str(block_replica)) return all_files = block_replica.block.files missing_files = all_files - block_replica.files() for lfile in missing_files: self.rlfsm.subscribe_file(block_replica.site, lfile) clone_block_replica = BlockReplica(block_replica.block, block_replica.site, block_replica.group) clone_block_replica.copy(block_replica) clone_block_replica.last_update = int(time.time()) clone_replica.block_replicas.add(clone_block_replica) if not self._read_only: for clone_replica in result: if clone_replica.growing: self.mysql.query('INSERT INTO `phedex_transfer_reservations` (`operation_id`, `item`, `site`, `group`) VALUES (%s, %s, %s, %s)', operation_id, clone_replica.dataset.name, clone_replica.site.name, clone_replica.group.name) else: for block_replica in clone_replica.block_replicas: self.mysql.query('INSERT INTO `phedex_transfer_reservations` (`operation_id`, `item`, `site`, `group`) VALUES (%s, %s, %s, %s)', operation_id, block_replica.block.full_name(), clone_replica.site.name, block_replica.group.name) # no external dependency - everything is a success return result
def schedule_deletions(self, replica_list, operation_id, comments = ''): #override LOG.info('Ignoring deletion schedule of %d replicas (operation %d)', len(replica_list), operation_id) result = [] for replica, block_replicas in replica_list: clone_replica = DatasetReplica(replica.dataset, replica.site) clone_replica.copy(replica) if block_replicas is None: result.append((clone_replica, None)) else: clone_block_replicas = [] for block_replica in block_replicas: clone_block_replica = BlockReplica(block_replica.block, block_replica.site, block_replica.group) clone_block_replica.copy(block_replica) clone_block_replica.last_update = int(time.time()) clone_block_replicas.append(clone_block_replica) result.append((clone_replica, clone_block_replicas)) return result
def schedule_deletions(self, replica_list, operation_id, comments = ''): #override sites = set(r.site for r, b in replica_list) if len(sites) != 1: raise OperationalError('schedule_copies should be called with a list of replicas at a single site.') site = list(sites)[0] LOG.info('Scheduling deletion of %d replicas from %s using RLFSM (operation %d)', len(replica_list), site.name, operation_id) clones = [] for dataset_replica, block_replicas in replica_list: if block_replicas is None: to_delete = dataset_replica.block_replicas else: to_delete = block_replicas for block_replica in to_delete: for lfile in block_replica.files(): self.rlfsm.desubscribe_file(block_replica.site, lfile) # No external dependency -> all operations are successful clone_replica = DatasetReplica(dataset_replica.dataset, dataset_replica.site) clone_replica.copy(dataset_replica) if block_replicas is None: clones.append((clone_replica, None)) else: clones.append((clone_replica, [])) for block_replica in block_replicas: clone_block_replica = BlockReplica(block_replica.block, block_replica.site) clone_block_replica.copy(block_replica) clone_block_replica.last_update = int(time.time()) clones[-1][1].append(clone_block_replica) return clones
def maker_blockreplicas(block, block_entry, site_check=None): """Return a list of block replicas using blockreplicas data or a combination of blockreplicas and filereplicas calls.""" sites = {} invalid_sites = set() groups = {} block_replicas = {} for replica_entry in block_entry['replica']: site_name = replica_entry['node'] try: site = sites[site_name] except KeyError: if site_check: if site_name in invalid_sites: continue if not site_check(site_name): invalid_sites.add(site_name) continue site = sites[site_name] = Site(site_name) group_name = replica_entry['group'] try: group = groups[group_name] except KeyError: group = groups[group_name] = Group(group_name) try: time_update = int(replica_entry['time_update']) except TypeError: # time_update was None time_update = 0 block_replica = BlockReplica( block, site, group, is_custodial=(replica_entry['custodial'] == 'y'), last_update=time_update) block_replicas[site_name] = block_replica if replica_entry['complete'] == 'n': # temporarily make this a list block_replica.file_ids = [] block_replica.size = 0 LOG.info("Incomplete %s" % str(block_replica)) if 'file' in block_entry: for file_entry in block_entry['file']: for replica_entry in file_entry['replica']: site_name = replica_entry['node'] try: block_replica = block_replicas[site_name] except KeyError: continue if block_replica.file_ids is None: continue # add LFN instead of file id block_replica.file_ids.append(file_entry['name']) file_size = file_entry['bytes'] if file_size is not None: block_replica.size += file_size try: time_create = int(replica_entry['time_create']) except TypeError: pass else: if time_create > block_replica.last_update: block_replica.last_update = time_create for block_replica in block_replicas.itervalues(): if block_replica.file_ids is not None: block_replica.file_ids = tuple(block_replica.file_ids) return block_replicas.values()
def schedule_deletions(self, replica_list, operation_id, comments=''): #override sites = set(r.site for r, b in replica_list) if len(sites) != 1: raise OperationalError( 'schedule_deletions should be called with a list of replicas at a single site.' ) site = list(sites)[0] LOG.info( 'Scheduling deletion of %d replicas from %s using RLFSM (operation %d)', len(replica_list), site.name, operation_id) clones = [] for dataset_replica, block_replicas in replica_list: if block_replicas is None: to_delete = dataset_replica.block_replicas else: to_delete = block_replicas for block_replica in to_delete: for lfile in block_replica.files(): self.rlfsm.desubscribe_file(block_replica.site, lfile) # No external dependency -> all operations are successful clone_replica = DatasetReplica(dataset_replica.dataset, dataset_replica.site) clone_replica.copy(dataset_replica) if block_replicas is None: clones.append((clone_replica, None)) else: clones.append((clone_replica, [])) for block_replica in block_replicas: clone_block_replica = BlockReplica(block_replica.block, block_replica.site, block_replica.group) clone_block_replica.copy(block_replica) clone_block_replica.last_update = int(time.time()) clones[-1][1].append(clone_block_replica) if not self._read_only: for clone_replica, block_replicas in clones: if block_replicas is None: self.mysql.query( 'INSERT INTO `phedex_deletion_reservations` (`operation_id`, `item`, `site`) VALUES (%s, %s, %s)', operation_id, clone_replica.dataset.name, clone_replica.site.name) else: for block_replica in block_replicas: self.mysql.query( 'INSERT INTO `phedex_deletion_reservations` (`operation_id`, `item`, `site`) VALUES (%s, %s, %s)', operation_id, block_replica.block.full_name(), clone_replica.site.name) return clones
def schedule_deletions(self, replica_list, operation_id, comments=''): #override sites = set(r.site for r, b in replica_list) if len(sites) != 1: raise OperationalError( 'schedule_copies should be called with a list of replicas at a single site.' ) site = list(sites)[0] if site.storage_type == Site.TYPE_MSS and not self.allow_tape_deletion: LOG.warning('Deletion from MSS not allowed by configuration.') return [] if self.allow_tape_deletion and self.auto_approval: LOG.warning( 'You cannot have auto-approved tape deletions. Set auto-approval to False.' ) return [] # execute the deletions in two steps: one for dataset-level and one for block-level datasets = [] blocks = [] # maps used later for cloning # getting ugly here.. should come up with a better way of making clones replica_map = {} block_replica_map = {} for dataset_replica, block_replicas in replica_list: if block_replicas is None: datasets.append(dataset_replica.dataset) else: blocks.extend(br.block for br in block_replicas) replica_map[dataset_replica.dataset] = dataset_replica block_replica_map.update( (br.block, br) for br in block_replicas) success = [] deleted_datasets = self._run_deletion_request(operation_id, site, 'dataset', datasets, comments) for dataset in deleted_datasets: replica = DatasetReplica(dataset, site, growing=False, group=Group.null_group) success.append((replica, None)) tmp_map = dict((dataset, []) for dataset in replica_map.iterkeys()) deleted_blocks = self._run_deletion_request(operation_id, site, 'block', blocks, comments) for block in deleted_blocks: tmp_map[block.dataset].append(block) for dataset, blocks in tmp_map.iteritems(): replica = DatasetReplica(dataset, site) replica.copy(replica_map[dataset]) success.append((replica, [])) for block in blocks: block_replica = BlockReplica(block, site, Group.null_group) block_replica.copy(block_replica_map[block]) block_replica.last_update = int(time.time()) success[-1][1].append(block_replica) return success