def run(self, caller, request, inventory):
self.parse_input(request, inventory, ('request_id',), ('request_id',))
request_id = self.params['request_id']
self.manager.lock()
try:
constraints = self.make_constraints(by_id = True)
existing_requests = self.manager.get_requests(**constraints)
if len(existing_requests) == 0:
raise InvalidRequest('Invalid request id %d' % request_id)
existing = existing_requests[request_id]
if existing.status == Request.ST_NEW:
existing.status = Request.ST_CANCELLED
self.update_request(existing)
elif existing.status == Request.ST_CANCELLED:
pass
else:
raise InvalidRequest('Request %d cannot be cancelled any more' % request_id)
finally:
self.manager.unlock()
return existing.to_dict()
def _validate_request(self, request, inventory, required, allowed=None):
for key in required:
if key not in request:
raise MissingParameter(key)
for key in request.iterkeys():
if key not in required and key not in allowed:
raise ExtraParameter(key)
if 'lockid' in request:
if type(request['lockid']) is str:
lock_ids = request['lockid'].split(',')
else:
lock_ids = request['lockid']
try:
request['lockid'] = map(int, lock_ids)
except ValueError:
raise InvalidRequest('Invalid lock id %s' % request['lockid'])
if 'sites' in request:
if type(request['sites']) is str:
request['sites'] = request['sites'].split(',')
for site in request['sites']:
if '*' in site or '?' in site:
pass
elif site not in inventory.sites:
raise InvalidRequest('Unknown site %s' % site)
if 'groups' in request:
if type(request['groups']) is str:
request['groups'] = request['groups'].split(',')
for group in request['groups']:
if '*' in group or '?' in group:
pass
elif group not in inventory.groups:
raise InvalidRequest('Unknown group %s' % group)
if 'user' in request:
if type(request['user']) is str:
request['user'] = request['user'].split(',')
for key in [
'expires', 'created_before', 'created_after', 'expires_before',
'expires_after'
]:
if key in request:
t = dateparser.parse(request[key])
request[key] = calendar.timegm(t.utctimetuple())
def _delete_datasets(self, objects, inventory, counts):
num_datasets = 0
for obj in objects:
try:
name = obj['name']
except KeyError:
raise MissingParameter('name', context='dataset ' + str(obj))
try:
dataset = inventory.datasets[name]
except KeyError:
raise InvalidRequest('Unknown dataset %s' % name)
if 'blocks' in obj:
# block-level deletion
self._delete_blocks(obj['blocks'], dataset, inventory, counts)
else:
try:
self._delete(inventory, dataset)
except:
raise RuntimeError('Inventory update failed')
num_datasets += 1
counts['datasets'] = num_datasets
def _delete_datasetreplicas(self, objects, inventory, counts):
num_datasetreplicas = 0
for obj in objects:
try:
dataset_name = obj['dataset']
except KeyError:
raise MissingParameter('dataset',
context='datasetreplica ' + str(obj))
else:
try:
dataset = inventory.datasets[dataset_name]
except KeyError:
raise InvalidRequest('Unknown dataset %s' % dataset_name)
try:
site_name = obj['site']
except KeyError:
raise MissingParameter('site',
context='datasetreplica ' + str(obj))
else:
try:
site = inventory.sites[site_name]
except KeyError:
raise InvalidRequest('Unknown site %s' % site_name)
replica = site.find_datast_replica(dataset)
if replica is None:
raise InvalidRequest('Replica of %s at %s does not exist' %
(dataset_name, site_name))
if 'blockreplicas' in obj:
# blockreplica-level deletion
self._delete_blockreplicas(obj['blockreplicas'], replica,
inventory, counts)
else:
# datasetreplica-level deletion
try:
self._delete(inventory, replica)
except:
raise RuntimeError('Inventory update failed')
num_datasetreplicas += 1
counts['datasetreplicas'] = num_datasetreplicas
def _delete_blockreplicas(self, objects, dataset_replica, inventory,
counts):
num_blockreplicas = 0
dataset = dataset_replica.dataset
site = dataset_replica.site
for obj in objects:
try:
block_name = obj['block']
except KeyError:
raise MissingParameter('block',
context='blockreplica ' + str(obj))
block_internal_name = df.Block.to_internal_name(block_name)
block = dataset.find_block(block_internal_name)
if block is None:
raise InvalidRequest('Unknown block %s' % block_name)
block_replica = block.find_replica(site)
if block_replica is None:
raise InvalidRequest('Replica of %s at %s does not exist' %
(block.full_name(), site.name))
try:
self._delete(inventory, block_replica)
except:
raise RuntimeError('Inventory update failed')
num_blockreplicas += 1
try:
counts['blockreplicas'] += num_blockreplicas
except KeyError:
counts['blockreplicas'] = num_blockreplicas
def _delete_files(self, objects, block, inventory, counts):
num_files = 0
updated_replicas = set()
for obj in objects:
try:
lfn = obj['name']
except KeyError:
raise MissingParameter('name', context='file ' + str(obj))
lfile = block.find_file(lfn)
if lfile is None:
raise InvalidRequest('Unknown file %s' % lfn)
for replica in block.replicas:
# delete_file adjusts the replica size too
if replica.delete_file(lfile, full_deletion=True):
updated_replicas.add(replica)
block.size -= lfile.size
block.num_files -= 1
try:
self._delete(inventory, lfile)
except:
raise RuntimeError('Inventory update failed')
num_files += 1
if num_files != 0:
self._register_update(inventory, block)
for replica in updated_replicas:
self._update(inventory, replica)
try:
counts['files'] += num_files
except KeyError:
counts['files'] = num_files
def _delete_sites(self, objects, inventory, counts):
num_sites = 0
for obj in objects:
try:
name = obj.pop('name')
except KeyError:
raise MissingParameter('name', context='site ' + str(obj))
try:
site = inventory.sites[name]
except KeyError:
raise InvalidRequest('Unknown site %s' % name)
try:
self._delete(inventory, site)
except:
raise RuntimeError('Inventory update failed')
num_sites += 1
counts['sites'] = num_sites
def _delete_groups(self, objects, inventory, counts):
num_groups = 0
for obj in objects:
try:
name = obj.pop('name')
except KeyError:
raise MissingParameter('name', context='group ' + str(obj))
try:
group = inventory.groups[name]
except KeyError:
raise InvalidRequest('Unknown group %s' % name)
try:
self._delete(inventory, group)
except:
raise RuntimeError('Inventory update failed')
num_groups += 1
counts['groups'] = num_groups
def _delete_blocks(self, objects, dataset, inventory, counts):
num_blocks = 0
for obj in objects:
try:
name = obj['name']
except KeyError:
raise MissingParameter('name', context='block ' + str(obj))
try:
internal_name = df.Block.to_internal_name(name)
except:
raise IllFormedRequest('name',
name,
hint='Name does not match the format')
block = dataset.find_block(internal_name)
if block is None:
raise InvalidRequest('Unknown block %s of %s' %
(name, dataset.name))
if 'files' in obj:
# file-level deletion
self._delete_files(obj['files'], block, inventory, counts)
else:
# block-level deletion
try:
self._delete(inventory, block)
except:
raise RuntimeError('Inventory update failed')
num_blocks += 1
try:
counts['blocks'] += num_blocks
except KeyError:
counts['blocks'] = num_blocks
def run(self, caller, request, inventory):
self.parse_input(request, inventory, ('request_id', 'item', 'site', 'group', 'n'))
self.manager.lock()
try:
existing = None
if 'request_id' in self.params:
request_id = self.params['request_id']
constraints = self.make_constraints(by_id = True)
existing_requests = self.manager.get_requests(**constraints)
if len(existing_requests) == 0:
raise InvalidRequest('Invalid request id %d' % request_id)
existing = existing_requests[request_id]
if existing.status != Request.ST_NEW:
raise InvalidRequest('Request %d cannot be updated any more' % request_id)
else:
# create a new request
if 'item' not in self.params:
raise MissingParameter('item')
if 'site' not in self.params:
if len(self.default_sites) == 0:
raise MissingParameter('site')
else:
self.params['site'] = list(self.default_sites)
constraints = self.make_constraints(by_id = False)
constraints['statuses'] = [Request.ST_NEW, Request.ST_ACTIVATED]
existing_requests = self.manager.get_requests(**constraints)
for request_id in sorted(existing_requests.iterkeys()):
if existing_requests[request_id].status == Request.ST_NEW:
existing = existing_requests[request_id]
break
elif existing_requests[request_id].status == Request.ST_ACTIVATED:
existing = existing_requests[request_id]
if existing is None:
if 'n' not in self.params:
self.params['n'] = 1
if 'group' not in self.params:
self.params['group'] = self.default_group
request = self.manager.create_request(caller, self.params['item'], self.params['site'], self.params['site_orig'], self.params['group'], self.params['n'])
else:
existing.request_count += 1
existing.last_request = int(time.time())
if existing.status == Request.ST_NEW:
# allow update of values
if 'group' in self.params:
existing.group = self.params['group']
if 'n' in self.params:
existing.n = self.params['n']
self.manager.update_request(existing)
request = existing
finally:
try:
self.manager.unlock()
except:
LOG.error('Error in manager.unlock()')
# requests is a single-element dictionary
return [request.to_dict()]
def parse_input(self,
request,
inventory,
allowed_fields,
required_fields=tuple()):
# JSON could have been uploaded
if self.input_data is not None:
request.update(self.input_data)
# Check we have the right request fields
input_fields = set(request.keys())
allowed_fields = set(allowed_fields)
excess = input_fields - allowed_fields
if len(excess) != 0:
raise ExtraParameter(list(excess)[0])
for key in required_fields:
if key not in request:
raise MissingParameter(key)
# Pick up the values and cast them to correct types
for key in ['request_id', 'n']:
if key not in request:
continue
try:
self.params[key] = int(request[key])
except ValueError:
raise IllFormedRequest(key,
request[key],
hint='%s must be an integer' % key)
for key in ['item', 'status', 'site', 'user']:
if key not in request:
continue
value = request[key]
if type(value) is str:
self.params[key] = value.strip().split(',')
elif type(value) is list:
self.params[key] = value
else:
raise IllFormedRequest(key,
request[key],
hint='%s must be a string or a list' %
key)
for key in ['group']:
if key not in request:
continue
self.params[key] = request[key]
for key in ['all']:
if key not in request:
continue
self.params[key] = yesno(request[key])
# Check value validity
# We check the site, group, and item names but not use their ids in the table.
# The only reason for this would be to make the registry not dependent on specific inventory store technology.
if 'item' in self.params:
for item in self.params['item']:
if item in inventory.datasets:
# OK this is a known dataset
continue
try:
dataset_name, block_name = df.Block.from_full_name(item)
except df.ObjectError:
raise InvalidRequest('Invalid item name %s' % item)
try:
inventory.datasets[dataset_name].find_block(block_name,
must_find=True)
except:
raise InvalidRequest('Invalid block name %s' % item)
if 'site' in self.params:
self.params['site_orig'] = []
for site in list(self.params['site']):
self.params['site_orig'].append(site)
# Wildcard allowed
if '*' in site or '?' in site or '[' in site:
self.params['site'].remove(site)
pattern = re.compile(fnmatch.translate(site))
for sname in inventory.sites.iterkeys():
if pattern.match(sname):
self.params['site'].append(sname)
else:
try:
inventory.sites[site]
except KeyError:
raise InvalidRequest('Invalid site name %s' % site)
if len(self.params['site']) == 0:
self.params.pop('site')
if 'group' in self.params:
try:
inventory.groups[self.params['group']]
except KeyError:
raise InvalidRequest('Invalid group name %s' %
self.params['group'])
if 'status' in self.params:
for status in self.params['status']:
if status not in ('new', 'activated', 'completed', 'rejected',
'cancelled'):
raise InvalidRequest('Invalid status value %s' % status)