def run(self, caller, request, inventory):
# Parse GET and POST requests and set the defaults
try:
statistic = request['statistic'].strip()
except:
statistic = 'size'
try:
list_by = request['list_by'].strip()
except:
list_by = next(cat for cat in InventoryStatCategories.categories.iterkeys())
constraints = {}
for key in InventoryStatCategories.categories.iterkeys():
try:
constraint = request[key]
except KeyError:
continue
if type(constraint) is str:
constraint = constraint.strip()
if ',' in constraint:
constraint = constraint.split(',')
constraints[key] = constraint
if len(constraints) == 0:
constraints = self.default_constraints
# HTML formatting
self.header_script = '$(document).ready(function() { initPage(\'%s\', \'%s\', %s); });' % (statistic, list_by, json.dumps(constraints))
repl = {}
categories_html = ''
constraint_types_html = ''
constraint_inputs_html = ''
for name, (title, _, _) in InventoryStatCategories.categories.iteritems():
categories_html += ' <option value="%s">%s</option>\n' % (name, title)
constraint_types_html += ' <div class="constraintType">%s</div>\n' % title
if name != 'group':
constraint_inputs_html += ' <div class="constraintInput"> = <input class="constraint" type="text" id="%s" name="%s"></div>' % (name, name)
repl['CATEGORIES'] = categories_html
repl['CONSTRAINT_TYPES'] = constraint_types_html
repl['CONSTRAINT_INPUTS'] = constraint_inputs_html
if yesno(request, 'physical', True):
repl['PHYSICAL_CHECKED'] = ' checked="checked"'
repl['PROJECTED_CHECKED'] = ''
else:
repl['PHYSICAL_CHECKED'] = ''
repl['PROJECTED_CHECKED'] = ' checked="checked"'
return self.form_html(repl)
def run(self, caller, request, inventory):
self._validate_request(request, inventory, [], ['lockid', 'user', 'service', 'item', 'sites', 'groups', 'created_before', 'created_after', 'expires_before', 'expires_after', 'showall'])
if 'user' not in request:
request['user'] = (caller.name,)
valid_only = (not yesno(request, 'showall', False))
existing = self._get_lock(request, valid_only = valid_only)
self.message = '%d locks found' % len(existing)
return existing
def run(self, caller, request, inventory):
"""
@return {'statistic': 'usage', 'content': [{'site': site_name, 'usage': [{key: key_name, size: size}]}]}
"""
if yesno(request, 'physical', True):
get_size = lambda bl: sum(br.size for br in bl)
else:
get_size = lambda bl: sum(br.block.size for br in bl)
all_replicas = filter_and_categorize(request, inventory)
by_site = {} # {site: {category: replicas}}
for category, replicas in all_replicas.iteritems():
for dataset_replica, block_replicas in replicas:
try:
site_content = by_site[dataset_replica.site]
except KeyError:
site_content = by_site[dataset_replica.site] = {}
try:
site_content[category].append(
(dataset_replica, block_replicas))
except KeyError:
site_content[category] = [(dataset_replica, block_replicas)
]
content = []
for site, site_replicas in by_site.iteritems():
site_content = []
for category, replicas in site_replicas.iteritems():
size = 0
for dataset_replica, block_replicas in replicas:
size += get_size(block_replicas)
site_content.append({'key': category, 'size': size * 1.e-12})
site_content.sort(key=lambda x: x['size'], reverse=True)
content.append({'site': site.name, 'usage': site_content})
content.sort(key=lambda x: x['site'])
return {
'statistic': 'usage',
'content': content,
'keys': sorted(all_replicas.keys())
}
def run(self, caller, request, inventory):
self._validate_request(request, inventory, [], [
'lockid', 'user', 'service', 'item', 'sites', 'groups',
'created_before', 'created_after', 'expires_before',
'expires_after', 'showall'
])
if 'user' not in request:
request['user'] = (caller.name, )
valid_only = (not yesno(request, 'showall', False))
existing = self._get_lock(request, valid_only=valid_only)
self.message = '%d locks found' % len(existing)
return existing
def run(self, caller, request, inventory):
"""
@return {'statistic': 'usage', 'content': [{'site': site_name, 'usage': [{key: key_name, size: size}]}]}
"""
if yesno(request, 'physical', True):
get_size = lambda bl: sum(br.size for br in bl)
else:
get_size = lambda bl: sum(br.block.size for br in bl)
all_replicas = filter_and_categorize(request, inventory)
by_site = {} # {site: {category: replicas}}
for category, replicas in all_replicas.iteritems():
for dataset_replica, block_replicas in replicas:
try:
site_content = by_site[dataset_replica.site]
except KeyError:
site_content = by_site[dataset_replica.site] = {}
try:
site_content[category].append((dataset_replica, block_replicas))
except KeyError:
site_content[category] = [(dataset_replica, block_replicas)]
content = []
for site, site_replicas in by_site.iteritems():
site_content = []
for category, replicas in site_replicas.iteritems():
size = 0
for dataset_replica, block_replicas in replicas:
size += get_size(block_replicas)
site_content.append({'key': category, 'size': size * 1.e-12})
site_content.sort(key = lambda x: x['size'], reverse = True)
content.append({'site': site.name, 'usage': site_content})
content.sort(key = lambda x: x['site'])
return {'statistic': 'usage', 'content': content, 'keys': sorted(all_replicas.keys())}
def get_partition_and_cycle(self, request, default_latest=True):
if 'partition' in request:
self.from_partition(request['partition'])
elif 'partition_id' in request:
try:
self.partition_id = int(request['partition_id'])
except ValueError:
raise exceptions.IllFormedRequest('partition_id',
request['partition_id'])
# partition_id can still be 0 here
requested_cycle = None
if 'cycle' in request:
try:
requested_cycle = int(request['cycle'])
except ValueError:
raise exceptions.IllFormedRequest('cycle', request['cycle'])
if requested_cycle != 0: # case 0 considered later
self.get_cycle(requested_cycle)
return requested_cycle
elif 'latest' in request and yesno(request, 'latest'):
requested_cycle = 0
# we either don't have a cycle request or are requested the latest cycle
if self.partition_id == 0:
self.from_partition()
if requested_cycle is not None or default_latest:
# requested_cycle must be 0 if not None
self.get_latest_cycle()
return requested_cycle
def get_partition_and_cycle(self, request, default_latest = True):
if 'partition' in request:
self.from_partition(request['partition'])
elif 'partition_id' in request:
try:
self.partition_id = int(request['partition_id'])
except ValueError:
raise exceptions.IllFormedRequest('partition_id', request['partition_id'])
# partition_id can still be 0 here
requested_cycle = None
if 'cycle' in request:
try:
requested_cycle = int(request['cycle'])
except ValueError:
raise exceptions.IllFormedRequest('cycle', request['cycle'])
if requested_cycle != 0: # case 0 considered later
self.get_cycle(requested_cycle)
return requested_cycle
elif 'latest' in request and yesno(request, 'latest'):
requested_cycle = 0
# we either don't have a cycle request or are requested the latest cycle
if self.partition_id == 0:
self.from_partition()
if requested_cycle is not None or default_latest:
# requested_cycle must be 0 if not None
self.get_latest_cycle()
return requested_cycle
def run(self, caller, request, inventory):
"""
@return {'statistic': 'size', 'content': [{key: key_name, size: size in TB}]}
"""
if yesno(request, 'physical'):
get_size = lambda bl: sum(br.size for br in bl)
else:
get_size = lambda bl: sum(br.block.size for br in bl)
all_replicas = filter_and_categorize(request, inventory)
content = []
for category, replicas in all_replicas.iteritems():
size = 0
for dataset_replica, block_replicas in replicas:
size += get_size(block_replicas)
content.append({'key': category, 'size': size * 1.e-12})
content.sort(key = lambda x: x['size'], reverse = True)
return {'statistic': 'size', 'content': content}
def run(self, caller, request, inventory):
"""
@return {'statistic': 'size', 'content': [{key: key_name, size: size in TB}]}
"""
if yesno(request, 'physical'):
get_size = lambda bl: sum(br.size for br in bl)
else:
get_size = lambda bl: sum(br.block.size for br in bl)
all_replicas = filter_and_categorize(request, inventory)
content = []
for category, replicas in all_replicas.iteritems():
size = 0
for dataset_replica, block_replicas in replicas:
size += get_size(block_replicas)
content.append({'key': category, 'size': size * 1.e-12})
content.sort(key=lambda x: x['size'], reverse=True)
return {'statistic': 'size', 'content': content}
def parse_input(self, request, inventory, allowed_fields, required_fields = tuple()):
# JSON could have been uploaded
if self.input_data is not None:
request.update(self.input_data)
# Check we have the right request fields
input_fields = set(request.keys())
allowed_fields = set(allowed_fields)
excess = input_fields - allowed_fields
if len(excess) != 0:
raise ExtraParameter(list(excess)[0])
for key in required_fields:
if key not in request:
raise MissingParameter(key)
# Pick up the values and cast them to correct types
for key in ['request_id', 'n']:
if key not in request:
continue
try:
self.params[key] = int(request[key])
except ValueError:
raise IllFormedRequest(key, request[key], hint = '%s must be an integer' % key)
for key in ['item', 'status', 'site', 'user']:
if key not in request:
continue
value = request[key]
if type(value) is str:
self.params[key] = value.strip().split(',')
elif type(value) is list:
self.params[key] = value
else:
raise IllFormedRequest(key, request[key], hint = '%s must be a string or a list' % key)
for key in ['group']:
if key not in request:
continue
self.params[key] = request[key]
for key in ['all']:
if key not in request:
continue
self.params[key] = yesno(request[key])
# Check value validity
# We check the site, group, and item names but not use their ids in the table.
# The only reason for this would be to make the registry not dependent on specific inventory store technology.
if 'item' in self.params:
for item in self.params['item']:
if item in inventory.datasets:
# OK this is a known dataset
continue
try:
dataset_name, block_name = df.Block.from_full_name(item)
except df.ObjectError:
raise InvalidRequest('Invalid item name %s' % item)
try:
inventory.datasets[dataset_name].find_block(block_name, must_find = True)
except:
raise InvalidRequest('Invalid block name %s' % item)
if 'site' in self.params:
self.params['site_orig'] = []
for site in list(self.params['site']):
self.params['site_orig'].append(site)
# Wildcard allowed
if '*' in site or '?' in site or '[' in site:
self.params['site'].remove(site)
pattern = re.compile(fnmatch.translate(site))
for sname in inventory.sites.iterkeys():
if pattern.match(sname):
self.params['site'].append(sname)
else:
try:
inventory.sites[site]
except KeyError:
raise InvalidRequest('Invalid site name %s' % site)
if len(self.params['site']) == 0:
self.params.pop('site')
if 'group' in self.params:
try:
inventory.groups[self.params['group']]
except KeyError:
raise InvalidRequest('Invalid group name %s' % self.params['group'])
if 'status' in self.params:
for status in self.params['status']:
if status not in ('new', 'activated', 'completed', 'rejected', 'cancelled'):
raise InvalidRequest('Invalid status value %s' % status)
def parse_input(self,
request,
inventory,
allowed_fields,
required_fields=tuple()):
# JSON could have been uploaded
if self.input_data is not None:
request.update(self.input_data)
# Check we have the right request fields
input_fields = set(request.keys())
allowed_fields = set(allowed_fields)
excess = input_fields - allowed_fields
if len(excess) != 0:
raise ExtraParameter(list(excess)[0])
for key in required_fields:
if key not in request:
raise MissingParameter(key)
# Pick up the values and cast them to correct types
for key in ['request_id', 'n']:
if key not in request:
continue
try:
self.params[key] = int(request[key])
except ValueError:
raise IllFormedRequest(key,
request[key],
hint='%s must be an integer' % key)
for key in ['item', 'status', 'site', 'user']:
if key not in request:
continue
value = request[key]
if type(value) is str:
self.params[key] = value.strip().split(',')
elif type(value) is list:
self.params[key] = value
else:
raise IllFormedRequest(key,
request[key],
hint='%s must be a string or a list' %
key)
for key in ['group']:
if key not in request:
continue
self.params[key] = request[key]
for key in ['all']:
if key not in request:
continue
self.params[key] = yesno(request[key])
# Check value validity
# We check the site, group, and item names but not use their ids in the table.
# The only reason for this would be to make the registry not dependent on specific inventory store technology.
if 'item' in self.params:
for item in self.params['item']:
if item in inventory.datasets:
# OK this is a known dataset
continue
try:
dataset_name, block_name = df.Block.from_full_name(item)
except df.ObjectError:
raise InvalidRequest('Invalid item name %s' % item)
try:
inventory.datasets[dataset_name].find_block(block_name,
must_find=True)
except:
raise InvalidRequest('Invalid block name %s' % item)
if 'site' in self.params:
self.params['site_orig'] = []
for site in list(self.params['site']):
self.params['site_orig'].append(site)
# Wildcard allowed
if '*' in site or '?' in site or '[' in site:
self.params['site'].remove(site)
pattern = re.compile(fnmatch.translate(site))
for sname in inventory.sites.iterkeys():
if pattern.match(sname):
self.params['site'].append(sname)
else:
try:
inventory.sites[site]
except KeyError:
raise InvalidRequest('Invalid site name %s' % site)
if len(self.params['site']) == 0:
self.params.pop('site')
if 'group' in self.params:
try:
inventory.groups[self.params['group']]
except KeyError:
raise InvalidRequest('Invalid group name %s' %
self.params['group'])
if 'status' in self.params:
for status in self.params['status']:
if status not in ('new', 'activated', 'completed', 'rejected',
'cancelled'):
raise InvalidRequest('Invalid status value %s' % status)
def run(self, caller, request, inventory):
# Parse GET and POST requests and set the defaults
try:
statistic = request['statistic'].strip()
except:
statistic = 'size'
try:
list_by = request['list_by'].strip()
except:
list_by = next(
cat for cat in InventoryStatCategories.categories.iterkeys())
constraints = {}
for key in InventoryStatCategories.categories.iterkeys():
try:
constraint = request[key]
except KeyError:
continue
if type(constraint) is str:
constraint = constraint.strip()
if ',' in constraint:
constraint = constraint.split(',')
constraints[key] = constraint
if len(constraints) == 0:
constraints = self.default_constraints
# HTML formatting
self.header_script = '$(document).ready(function() { initPage(\'%s\', \'%s\', %s); });' % (
statistic, list_by, json.dumps(constraints))
repl = {}
categories_html = ''
constraint_types_html = ''
constraint_inputs_html = ''
for name, (title, _,
_) in InventoryStatCategories.categories.iteritems():
categories_html += ' <option value="%s">%s</option>\n' % (
name, title)
constraint_types_html += ' <div class="constraintType">%s</div>\n' % title
if name != 'group':
constraint_inputs_html += ' <div class="constraintInput"> = <input class="constraint" type="text" id="%s" name="%s"></div>' % (
name, name)
repl['CATEGORIES'] = categories_html
repl['CONSTRAINT_TYPES'] = constraint_types_html
repl['CONSTRAINT_INPUTS'] = constraint_inputs_html
if yesno(request, 'physical', True):
repl['PHYSICAL_CHECKED'] = ' checked="checked"'
repl['PROJECTED_CHECKED'] = ''
else:
repl['PHYSICAL_CHECKED'] = ''
repl['PROJECTED_CHECKED'] = ' checked="checked"'
return self.form_html(repl)