def run(self, caller, request, inventory): self.parse_input(request, inventory, ('request_id',), ('request_id',)) request_id = self.params['request_id'] self.manager.lock() try: constraints = self.make_constraints(by_id = True) existing_requests = self.manager.get_requests(**constraints) if len(existing_requests) == 0: raise InvalidRequest('Invalid request id %d' % request_id) existing = existing_requests[request_id] if existing.status == Request.ST_NEW: existing.status = Request.ST_CANCELLED self.update_request(existing) elif existing.status == Request.ST_CANCELLED: pass else: raise InvalidRequest('Request %d cannot be cancelled any more' % request_id) finally: self.manager.unlock() return existing.to_dict()
def _validate_request(self, request, inventory, required, allowed=None): for key in required: if key not in request: raise MissingParameter(key) for key in request.iterkeys(): if key not in required and key not in allowed: raise ExtraParameter(key) if 'lockid' in request: if type(request['lockid']) is str: lock_ids = request['lockid'].split(',') else: lock_ids = request['lockid'] try: request['lockid'] = map(int, lock_ids) except ValueError: raise InvalidRequest('Invalid lock id %s' % request['lockid']) if 'sites' in request: if type(request['sites']) is str: request['sites'] = request['sites'].split(',') for site in request['sites']: if '*' in site or '?' in site: pass elif site not in inventory.sites: raise InvalidRequest('Unknown site %s' % site) if 'groups' in request: if type(request['groups']) is str: request['groups'] = request['groups'].split(',') for group in request['groups']: if '*' in group or '?' in group: pass elif group not in inventory.groups: raise InvalidRequest('Unknown group %s' % group) if 'user' in request: if type(request['user']) is str: request['user'] = request['user'].split(',') for key in [ 'expires', 'created_before', 'created_after', 'expires_before', 'expires_after' ]: if key in request: t = dateparser.parse(request[key]) request[key] = calendar.timegm(t.utctimetuple())
def _delete_datasets(self, objects, inventory, counts): num_datasets = 0 for obj in objects: try: name = obj['name'] except KeyError: raise MissingParameter('name', context='dataset ' + str(obj)) try: dataset = inventory.datasets[name] except KeyError: raise InvalidRequest('Unknown dataset %s' % name) if 'blocks' in obj: # block-level deletion self._delete_blocks(obj['blocks'], dataset, inventory, counts) else: try: self._delete(inventory, dataset) except: raise RuntimeError('Inventory update failed') num_datasets += 1 counts['datasets'] = num_datasets
def _delete_datasetreplicas(self, objects, inventory, counts): num_datasetreplicas = 0 for obj in objects: try: dataset_name = obj['dataset'] except KeyError: raise MissingParameter('dataset', context='datasetreplica ' + str(obj)) else: try: dataset = inventory.datasets[dataset_name] except KeyError: raise InvalidRequest('Unknown dataset %s' % dataset_name) try: site_name = obj['site'] except KeyError: raise MissingParameter('site', context='datasetreplica ' + str(obj)) else: try: site = inventory.sites[site_name] except KeyError: raise InvalidRequest('Unknown site %s' % site_name) replica = site.find_datast_replica(dataset) if replica is None: raise InvalidRequest('Replica of %s at %s does not exist' % (dataset_name, site_name)) if 'blockreplicas' in obj: # blockreplica-level deletion self._delete_blockreplicas(obj['blockreplicas'], replica, inventory, counts) else: # datasetreplica-level deletion try: self._delete(inventory, replica) except: raise RuntimeError('Inventory update failed') num_datasetreplicas += 1 counts['datasetreplicas'] = num_datasetreplicas
def _delete_blockreplicas(self, objects, dataset_replica, inventory, counts): num_blockreplicas = 0 dataset = dataset_replica.dataset site = dataset_replica.site for obj in objects: try: block_name = obj['block'] except KeyError: raise MissingParameter('block', context='blockreplica ' + str(obj)) block_internal_name = df.Block.to_internal_name(block_name) block = dataset.find_block(block_internal_name) if block is None: raise InvalidRequest('Unknown block %s' % block_name) block_replica = block.find_replica(site) if block_replica is None: raise InvalidRequest('Replica of %s at %s does not exist' % (block.full_name(), site.name)) try: self._delete(inventory, block_replica) except: raise RuntimeError('Inventory update failed') num_blockreplicas += 1 try: counts['blockreplicas'] += num_blockreplicas except KeyError: counts['blockreplicas'] = num_blockreplicas
def _delete_files(self, objects, block, inventory, counts): num_files = 0 updated_replicas = set() for obj in objects: try: lfn = obj['name'] except KeyError: raise MissingParameter('name', context='file ' + str(obj)) lfile = block.find_file(lfn) if lfile is None: raise InvalidRequest('Unknown file %s' % lfn) for replica in block.replicas: # delete_file adjusts the replica size too if replica.delete_file(lfile, full_deletion=True): updated_replicas.add(replica) block.size -= lfile.size block.num_files -= 1 try: self._delete(inventory, lfile) except: raise RuntimeError('Inventory update failed') num_files += 1 if num_files != 0: self._register_update(inventory, block) for replica in updated_replicas: self._update(inventory, replica) try: counts['files'] += num_files except KeyError: counts['files'] = num_files
def _delete_sites(self, objects, inventory, counts): num_sites = 0 for obj in objects: try: name = obj.pop('name') except KeyError: raise MissingParameter('name', context='site ' + str(obj)) try: site = inventory.sites[name] except KeyError: raise InvalidRequest('Unknown site %s' % name) try: self._delete(inventory, site) except: raise RuntimeError('Inventory update failed') num_sites += 1 counts['sites'] = num_sites
def _delete_groups(self, objects, inventory, counts): num_groups = 0 for obj in objects: try: name = obj.pop('name') except KeyError: raise MissingParameter('name', context='group ' + str(obj)) try: group = inventory.groups[name] except KeyError: raise InvalidRequest('Unknown group %s' % name) try: self._delete(inventory, group) except: raise RuntimeError('Inventory update failed') num_groups += 1 counts['groups'] = num_groups
def _delete_blocks(self, objects, dataset, inventory, counts): num_blocks = 0 for obj in objects: try: name = obj['name'] except KeyError: raise MissingParameter('name', context='block ' + str(obj)) try: internal_name = df.Block.to_internal_name(name) except: raise IllFormedRequest('name', name, hint='Name does not match the format') block = dataset.find_block(internal_name) if block is None: raise InvalidRequest('Unknown block %s of %s' % (name, dataset.name)) if 'files' in obj: # file-level deletion self._delete_files(obj['files'], block, inventory, counts) else: # block-level deletion try: self._delete(inventory, block) except: raise RuntimeError('Inventory update failed') num_blocks += 1 try: counts['blocks'] += num_blocks except KeyError: counts['blocks'] = num_blocks
def run(self, caller, request, inventory): self.parse_input(request, inventory, ('request_id', 'item', 'site', 'group', 'n')) self.manager.lock() try: existing = None if 'request_id' in self.params: request_id = self.params['request_id'] constraints = self.make_constraints(by_id = True) existing_requests = self.manager.get_requests(**constraints) if len(existing_requests) == 0: raise InvalidRequest('Invalid request id %d' % request_id) existing = existing_requests[request_id] if existing.status != Request.ST_NEW: raise InvalidRequest('Request %d cannot be updated any more' % request_id) else: # create a new request if 'item' not in self.params: raise MissingParameter('item') if 'site' not in self.params: if len(self.default_sites) == 0: raise MissingParameter('site') else: self.params['site'] = list(self.default_sites) constraints = self.make_constraints(by_id = False) constraints['statuses'] = [Request.ST_NEW, Request.ST_ACTIVATED] existing_requests = self.manager.get_requests(**constraints) for request_id in sorted(existing_requests.iterkeys()): if existing_requests[request_id].status == Request.ST_NEW: existing = existing_requests[request_id] break elif existing_requests[request_id].status == Request.ST_ACTIVATED: existing = existing_requests[request_id] if existing is None: if 'n' not in self.params: self.params['n'] = 1 if 'group' not in self.params: self.params['group'] = self.default_group request = self.manager.create_request(caller, self.params['item'], self.params['site'], self.params['site_orig'], self.params['group'], self.params['n']) else: existing.request_count += 1 existing.last_request = int(time.time()) if existing.status == Request.ST_NEW: # allow update of values if 'group' in self.params: existing.group = self.params['group'] if 'n' in self.params: existing.n = self.params['n'] self.manager.update_request(existing) request = existing finally: try: self.manager.unlock() except: LOG.error('Error in manager.unlock()') # requests is a single-element dictionary return [request.to_dict()]
def parse_input(self, request, inventory, allowed_fields, required_fields=tuple()): # JSON could have been uploaded if self.input_data is not None: request.update(self.input_data) # Check we have the right request fields input_fields = set(request.keys()) allowed_fields = set(allowed_fields) excess = input_fields - allowed_fields if len(excess) != 0: raise ExtraParameter(list(excess)[0]) for key in required_fields: if key not in request: raise MissingParameter(key) # Pick up the values and cast them to correct types for key in ['request_id', 'n']: if key not in request: continue try: self.params[key] = int(request[key]) except ValueError: raise IllFormedRequest(key, request[key], hint='%s must be an integer' % key) for key in ['item', 'status', 'site', 'user']: if key not in request: continue value = request[key] if type(value) is str: self.params[key] = value.strip().split(',') elif type(value) is list: self.params[key] = value else: raise IllFormedRequest(key, request[key], hint='%s must be a string or a list' % key) for key in ['group']: if key not in request: continue self.params[key] = request[key] for key in ['all']: if key not in request: continue self.params[key] = yesno(request[key]) # Check value validity # We check the site, group, and item names but not use their ids in the table. # The only reason for this would be to make the registry not dependent on specific inventory store technology. if 'item' in self.params: for item in self.params['item']: if item in inventory.datasets: # OK this is a known dataset continue try: dataset_name, block_name = df.Block.from_full_name(item) except df.ObjectError: raise InvalidRequest('Invalid item name %s' % item) try: inventory.datasets[dataset_name].find_block(block_name, must_find=True) except: raise InvalidRequest('Invalid block name %s' % item) if 'site' in self.params: self.params['site_orig'] = [] for site in list(self.params['site']): self.params['site_orig'].append(site) # Wildcard allowed if '*' in site or '?' in site or '[' in site: self.params['site'].remove(site) pattern = re.compile(fnmatch.translate(site)) for sname in inventory.sites.iterkeys(): if pattern.match(sname): self.params['site'].append(sname) else: try: inventory.sites[site] except KeyError: raise InvalidRequest('Invalid site name %s' % site) if len(self.params['site']) == 0: self.params.pop('site') if 'group' in self.params: try: inventory.groups[self.params['group']] except KeyError: raise InvalidRequest('Invalid group name %s' % self.params['group']) if 'status' in self.params: for status in self.params['status']: if status not in ('new', 'activated', 'completed', 'rejected', 'cancelled'): raise InvalidRequest('Invalid status value %s' % status)