def run(self, caller, request, inventory): # Parse GET and POST requests and set the defaults try: statistic = request['statistic'].strip() except: statistic = 'size' try: list_by = request['list_by'].strip() except: list_by = next(cat for cat in InventoryStatCategories.categories.iterkeys()) constraints = {} for key in InventoryStatCategories.categories.iterkeys(): try: constraint = request[key] except KeyError: continue if type(constraint) is str: constraint = constraint.strip() if ',' in constraint: constraint = constraint.split(',') constraints[key] = constraint if len(constraints) == 0: constraints = self.default_constraints # HTML formatting self.header_script = '$(document).ready(function() { initPage(\'%s\', \'%s\', %s); });' % (statistic, list_by, json.dumps(constraints)) repl = {} categories_html = '' constraint_types_html = '' constraint_inputs_html = '' for name, (title, _, _) in InventoryStatCategories.categories.iteritems(): categories_html += ' <option value="%s">%s</option>\n' % (name, title) constraint_types_html += ' <div class="constraintType">%s</div>\n' % title if name != 'group': constraint_inputs_html += ' <div class="constraintInput"> = <input class="constraint" type="text" id="%s" name="%s"></div>' % (name, name) repl['CATEGORIES'] = categories_html repl['CONSTRAINT_TYPES'] = constraint_types_html repl['CONSTRAINT_INPUTS'] = constraint_inputs_html if yesno(request, 'physical', True): repl['PHYSICAL_CHECKED'] = ' checked="checked"' repl['PROJECTED_CHECKED'] = '' else: repl['PHYSICAL_CHECKED'] = '' repl['PROJECTED_CHECKED'] = ' checked="checked"' return self.form_html(repl)
def run(self, caller, request, inventory): self._validate_request(request, inventory, [], ['lockid', 'user', 'service', 'item', 'sites', 'groups', 'created_before', 'created_after', 'expires_before', 'expires_after', 'showall']) if 'user' not in request: request['user'] = (caller.name,) valid_only = (not yesno(request, 'showall', False)) existing = self._get_lock(request, valid_only = valid_only) self.message = '%d locks found' % len(existing) return existing
def run(self, caller, request, inventory): """ @return {'statistic': 'usage', 'content': [{'site': site_name, 'usage': [{key: key_name, size: size}]}]} """ if yesno(request, 'physical', True): get_size = lambda bl: sum(br.size for br in bl) else: get_size = lambda bl: sum(br.block.size for br in bl) all_replicas = filter_and_categorize(request, inventory) by_site = {} # {site: {category: replicas}} for category, replicas in all_replicas.iteritems(): for dataset_replica, block_replicas in replicas: try: site_content = by_site[dataset_replica.site] except KeyError: site_content = by_site[dataset_replica.site] = {} try: site_content[category].append( (dataset_replica, block_replicas)) except KeyError: site_content[category] = [(dataset_replica, block_replicas) ] content = [] for site, site_replicas in by_site.iteritems(): site_content = [] for category, replicas in site_replicas.iteritems(): size = 0 for dataset_replica, block_replicas in replicas: size += get_size(block_replicas) site_content.append({'key': category, 'size': size * 1.e-12}) site_content.sort(key=lambda x: x['size'], reverse=True) content.append({'site': site.name, 'usage': site_content}) content.sort(key=lambda x: x['site']) return { 'statistic': 'usage', 'content': content, 'keys': sorted(all_replicas.keys()) }
def run(self, caller, request, inventory): self._validate_request(request, inventory, [], [ 'lockid', 'user', 'service', 'item', 'sites', 'groups', 'created_before', 'created_after', 'expires_before', 'expires_after', 'showall' ]) if 'user' not in request: request['user'] = (caller.name, ) valid_only = (not yesno(request, 'showall', False)) existing = self._get_lock(request, valid_only=valid_only) self.message = '%d locks found' % len(existing) return existing
def run(self, caller, request, inventory): """ @return {'statistic': 'usage', 'content': [{'site': site_name, 'usage': [{key: key_name, size: size}]}]} """ if yesno(request, 'physical', True): get_size = lambda bl: sum(br.size for br in bl) else: get_size = lambda bl: sum(br.block.size for br in bl) all_replicas = filter_and_categorize(request, inventory) by_site = {} # {site: {category: replicas}} for category, replicas in all_replicas.iteritems(): for dataset_replica, block_replicas in replicas: try: site_content = by_site[dataset_replica.site] except KeyError: site_content = by_site[dataset_replica.site] = {} try: site_content[category].append((dataset_replica, block_replicas)) except KeyError: site_content[category] = [(dataset_replica, block_replicas)] content = [] for site, site_replicas in by_site.iteritems(): site_content = [] for category, replicas in site_replicas.iteritems(): size = 0 for dataset_replica, block_replicas in replicas: size += get_size(block_replicas) site_content.append({'key': category, 'size': size * 1.e-12}) site_content.sort(key = lambda x: x['size'], reverse = True) content.append({'site': site.name, 'usage': site_content}) content.sort(key = lambda x: x['site']) return {'statistic': 'usage', 'content': content, 'keys': sorted(all_replicas.keys())}
def get_partition_and_cycle(self, request, default_latest=True): if 'partition' in request: self.from_partition(request['partition']) elif 'partition_id' in request: try: self.partition_id = int(request['partition_id']) except ValueError: raise exceptions.IllFormedRequest('partition_id', request['partition_id']) # partition_id can still be 0 here requested_cycle = None if 'cycle' in request: try: requested_cycle = int(request['cycle']) except ValueError: raise exceptions.IllFormedRequest('cycle', request['cycle']) if requested_cycle != 0: # case 0 considered later self.get_cycle(requested_cycle) return requested_cycle elif 'latest' in request and yesno(request, 'latest'): requested_cycle = 0 # we either don't have a cycle request or are requested the latest cycle if self.partition_id == 0: self.from_partition() if requested_cycle is not None or default_latest: # requested_cycle must be 0 if not None self.get_latest_cycle() return requested_cycle
def get_partition_and_cycle(self, request, default_latest = True): if 'partition' in request: self.from_partition(request['partition']) elif 'partition_id' in request: try: self.partition_id = int(request['partition_id']) except ValueError: raise exceptions.IllFormedRequest('partition_id', request['partition_id']) # partition_id can still be 0 here requested_cycle = None if 'cycle' in request: try: requested_cycle = int(request['cycle']) except ValueError: raise exceptions.IllFormedRequest('cycle', request['cycle']) if requested_cycle != 0: # case 0 considered later self.get_cycle(requested_cycle) return requested_cycle elif 'latest' in request and yesno(request, 'latest'): requested_cycle = 0 # we either don't have a cycle request or are requested the latest cycle if self.partition_id == 0: self.from_partition() if requested_cycle is not None or default_latest: # requested_cycle must be 0 if not None self.get_latest_cycle() return requested_cycle
def run(self, caller, request, inventory): """ @return {'statistic': 'size', 'content': [{key: key_name, size: size in TB}]} """ if yesno(request, 'physical'): get_size = lambda bl: sum(br.size for br in bl) else: get_size = lambda bl: sum(br.block.size for br in bl) all_replicas = filter_and_categorize(request, inventory) content = [] for category, replicas in all_replicas.iteritems(): size = 0 for dataset_replica, block_replicas in replicas: size += get_size(block_replicas) content.append({'key': category, 'size': size * 1.e-12}) content.sort(key = lambda x: x['size'], reverse = True) return {'statistic': 'size', 'content': content}
def run(self, caller, request, inventory): """ @return {'statistic': 'size', 'content': [{key: key_name, size: size in TB}]} """ if yesno(request, 'physical'): get_size = lambda bl: sum(br.size for br in bl) else: get_size = lambda bl: sum(br.block.size for br in bl) all_replicas = filter_and_categorize(request, inventory) content = [] for category, replicas in all_replicas.iteritems(): size = 0 for dataset_replica, block_replicas in replicas: size += get_size(block_replicas) content.append({'key': category, 'size': size * 1.e-12}) content.sort(key=lambda x: x['size'], reverse=True) return {'statistic': 'size', 'content': content}
def parse_input(self, request, inventory, allowed_fields, required_fields = tuple()): # JSON could have been uploaded if self.input_data is not None: request.update(self.input_data) # Check we have the right request fields input_fields = set(request.keys()) allowed_fields = set(allowed_fields) excess = input_fields - allowed_fields if len(excess) != 0: raise ExtraParameter(list(excess)[0]) for key in required_fields: if key not in request: raise MissingParameter(key) # Pick up the values and cast them to correct types for key in ['request_id', 'n']: if key not in request: continue try: self.params[key] = int(request[key]) except ValueError: raise IllFormedRequest(key, request[key], hint = '%s must be an integer' % key) for key in ['item', 'status', 'site', 'user']: if key not in request: continue value = request[key] if type(value) is str: self.params[key] = value.strip().split(',') elif type(value) is list: self.params[key] = value else: raise IllFormedRequest(key, request[key], hint = '%s must be a string or a list' % key) for key in ['group']: if key not in request: continue self.params[key] = request[key] for key in ['all']: if key not in request: continue self.params[key] = yesno(request[key]) # Check value validity # We check the site, group, and item names but not use their ids in the table. # The only reason for this would be to make the registry not dependent on specific inventory store technology. if 'item' in self.params: for item in self.params['item']: if item in inventory.datasets: # OK this is a known dataset continue try: dataset_name, block_name = df.Block.from_full_name(item) except df.ObjectError: raise InvalidRequest('Invalid item name %s' % item) try: inventory.datasets[dataset_name].find_block(block_name, must_find = True) except: raise InvalidRequest('Invalid block name %s' % item) if 'site' in self.params: self.params['site_orig'] = [] for site in list(self.params['site']): self.params['site_orig'].append(site) # Wildcard allowed if '*' in site or '?' in site or '[' in site: self.params['site'].remove(site) pattern = re.compile(fnmatch.translate(site)) for sname in inventory.sites.iterkeys(): if pattern.match(sname): self.params['site'].append(sname) else: try: inventory.sites[site] except KeyError: raise InvalidRequest('Invalid site name %s' % site) if len(self.params['site']) == 0: self.params.pop('site') if 'group' in self.params: try: inventory.groups[self.params['group']] except KeyError: raise InvalidRequest('Invalid group name %s' % self.params['group']) if 'status' in self.params: for status in self.params['status']: if status not in ('new', 'activated', 'completed', 'rejected', 'cancelled'): raise InvalidRequest('Invalid status value %s' % status)
def parse_input(self, request, inventory, allowed_fields, required_fields=tuple()): # JSON could have been uploaded if self.input_data is not None: request.update(self.input_data) # Check we have the right request fields input_fields = set(request.keys()) allowed_fields = set(allowed_fields) excess = input_fields - allowed_fields if len(excess) != 0: raise ExtraParameter(list(excess)[0]) for key in required_fields: if key not in request: raise MissingParameter(key) # Pick up the values and cast them to correct types for key in ['request_id', 'n']: if key not in request: continue try: self.params[key] = int(request[key]) except ValueError: raise IllFormedRequest(key, request[key], hint='%s must be an integer' % key) for key in ['item', 'status', 'site', 'user']: if key not in request: continue value = request[key] if type(value) is str: self.params[key] = value.strip().split(',') elif type(value) is list: self.params[key] = value else: raise IllFormedRequest(key, request[key], hint='%s must be a string or a list' % key) for key in ['group']: if key not in request: continue self.params[key] = request[key] for key in ['all']: if key not in request: continue self.params[key] = yesno(request[key]) # Check value validity # We check the site, group, and item names but not use their ids in the table. # The only reason for this would be to make the registry not dependent on specific inventory store technology. if 'item' in self.params: for item in self.params['item']: if item in inventory.datasets: # OK this is a known dataset continue try: dataset_name, block_name = df.Block.from_full_name(item) except df.ObjectError: raise InvalidRequest('Invalid item name %s' % item) try: inventory.datasets[dataset_name].find_block(block_name, must_find=True) except: raise InvalidRequest('Invalid block name %s' % item) if 'site' in self.params: self.params['site_orig'] = [] for site in list(self.params['site']): self.params['site_orig'].append(site) # Wildcard allowed if '*' in site or '?' in site or '[' in site: self.params['site'].remove(site) pattern = re.compile(fnmatch.translate(site)) for sname in inventory.sites.iterkeys(): if pattern.match(sname): self.params['site'].append(sname) else: try: inventory.sites[site] except KeyError: raise InvalidRequest('Invalid site name %s' % site) if len(self.params['site']) == 0: self.params.pop('site') if 'group' in self.params: try: inventory.groups[self.params['group']] except KeyError: raise InvalidRequest('Invalid group name %s' % self.params['group']) if 'status' in self.params: for status in self.params['status']: if status not in ('new', 'activated', 'completed', 'rejected', 'cancelled'): raise InvalidRequest('Invalid status value %s' % status)
def run(self, caller, request, inventory): # Parse GET and POST requests and set the defaults try: statistic = request['statistic'].strip() except: statistic = 'size' try: list_by = request['list_by'].strip() except: list_by = next( cat for cat in InventoryStatCategories.categories.iterkeys()) constraints = {} for key in InventoryStatCategories.categories.iterkeys(): try: constraint = request[key] except KeyError: continue if type(constraint) is str: constraint = constraint.strip() if ',' in constraint: constraint = constraint.split(',') constraints[key] = constraint if len(constraints) == 0: constraints = self.default_constraints # HTML formatting self.header_script = '$(document).ready(function() { initPage(\'%s\', \'%s\', %s); });' % ( statistic, list_by, json.dumps(constraints)) repl = {} categories_html = '' constraint_types_html = '' constraint_inputs_html = '' for name, (title, _, _) in InventoryStatCategories.categories.iteritems(): categories_html += ' <option value="%s">%s</option>\n' % ( name, title) constraint_types_html += ' <div class="constraintType">%s</div>\n' % title if name != 'group': constraint_inputs_html += ' <div class="constraintInput"> = <input class="constraint" type="text" id="%s" name="%s"></div>' % ( name, name) repl['CATEGORIES'] = categories_html repl['CONSTRAINT_TYPES'] = constraint_types_html repl['CONSTRAINT_INPUTS'] = constraint_inputs_html if yesno(request, 'physical', True): repl['PHYSICAL_CHECKED'] = ' checked="checked"' repl['PROJECTED_CHECKED'] = '' else: repl['PHYSICAL_CHECKED'] = '' repl['PROJECTED_CHECKED'] = ' checked="checked"' return self.form_html(repl)