def add(request, label_id, version): """Add this regulation node and all of its children to the db""" try: node = json.loads(request.body) jsonschema.validate(node, REGULATION_SCHEMA) except ValueError: return user_error('invalid format') except jsonschema.ValidationError: return user_error("JSON is invalid") if label_id != '-'.join(node['label']): return user_error('label mismatch') to_save = [] labels_seen = set() def add_node(node): label_tuple = tuple(node['label']) if label_tuple in labels_seen: logging.warning("Repeat label: %s", label_tuple) labels_seen.add(label_tuple) node = dict(node) # copy to_save.append(node) for child in node['children']: add_node(child) add_node(node) db.Regulations().bulk_put(to_save, version, label_id) return success()
def search(request, doc_type): """Search elastic search for any matches in the node's text""" term = request.GET.get('q', '') version = request.GET.get('version', '') regulation = request.GET.get('regulation', '') is_root = request.GET.get('is_root') is_subpart = request.GET.get('is_subpart') try: page = int(request.GET.get('page', '0')) except ValueError: page = 0 if not term: return user_error('No query term') if not validate_boolean(is_root): return user_error('Parameter "is_root" must be "true" or "false"') if not validate_boolean(is_subpart): return user_error('Parameter "is_subpart" must be "true" or "false"') query = { 'fields': ['text', 'label', 'version', 'regulation', 'title', 'label_string'], 'from': page * PAGE_SIZE, 'size': PAGE_SIZE, } text_match = {'match': {'text': term, 'doc_type': doc_type}} if version or regulation: term = {} if version: term['version'] = version if regulation: term['regulation'] = regulation if is_root: term['is_root'] = is_root if is_subpart: term['is_subpart'] = is_subpart query['query'] = { 'filtered': { 'query': text_match, 'filter': { 'term': term } } } else: query['query'] = text_match es = ElasticSearch(settings.ELASTIC_SEARCH_URLS) results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX) return success({ 'total_hits': results['hits']['total'], 'results': transform_results([h['fields'] for h in results['hits']['hits']]) })
def add(request, label_id, version): """Add this regulation node and all of its children to the db""" try: node = json.loads(request.body) jsonschema.validate(node, REGULATION_SCHEMA) except ValueError: return user_error('invalid format') except jsonschema.ValidationError, e: return user_error("JSON is invalid")
def add(request, label_id, version): """Add this regulation node and all of its children to the db""" try: node = anyjson.deserialize(request.body) jsonschema.validate(node, REGULATION_SCHEMA) except ValueError: return user_error('invalid format') except jsonschema.ValidationError, e: return user_error("JSON is invalid")
def add(request, doc_type, label_id, version=None): """Add this document node and all of its children to the db""" try: node = request.json_body jsonschema.validate(node, REGULATION_SCHEMA) except jsonschema.ValidationError: return user_error("JSON is invalid") if label_id != '-'.join(node['label']): return user_error('label mismatch') write_node(node, doc_type, label_id, version) return success()
def add(request, doc_type, label_id, version=None): """Add this document node and all of its children to the db""" try: node = request.json_body jsonschema.validate(node, REGULATION_SCHEMA) except jsonschema.ValidationError: return user_error("JSON is invalid") if label_id != '-'.join(node['label']): return user_error('label mismatch') write_node(node, doc_type, label_id, version) return success()
def search(request, doc_type): """Search elastic search for any matches in the node's text""" term = request.GET.get('q', '') version = request.GET.get('version', '') regulation = request.GET.get('regulation', '') is_root = request.GET.get('is_root') is_subpart = request.GET.get('is_subpart') try: page = int(request.GET.get('page', '0')) except ValueError: page = 0 if not term: return user_error('No query term') if not validate_boolean(is_root): return user_error('Parameter "is_root" must be "true" or "false"') if not validate_boolean(is_subpart): return user_error('Parameter "is_subpart" must be "true" or "false"') query = { 'fields': ['text', 'label', 'version', 'regulation', 'title', 'label_string'], 'from': page * PAGE_SIZE, 'size': PAGE_SIZE, } text_match = {'match': {'text': term, 'doc_type': doc_type}} if version or regulation: term = {} if version: term['version'] = version if regulation: term['regulation'] = regulation if is_root: term['is_root'] = is_root if is_subpart: term['is_subpart'] = is_subpart query['query'] = {'filtered': { 'query': text_match, 'filter': {'term': term} }} else: query['query'] = text_match es = ElasticSearch(settings.ELASTIC_SEARCH_URLS) results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX) return success({ 'total_hits': results['hits']['total'], 'results': transform_results([h['fields'] for h in results['hits']['hits']]) })
def search(request): """Use haystack to find search results""" term = request.GET.get('q', '') version = request.GET.get('version', '') regulation = request.GET.get('regulation', '') try: page = int(request.GET.get('page', '0')) except ValueError: page = 0 if not term: return user_error('No query term') query = SearchQuerySet().models(Regulation).filter(content=term) if version: query = query.filter(version=version) if regulation: query = query.filter(regulation=regulation) start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE return success({ 'total_hits': len(query), 'results': transform_results(query[start:end]) })
def wrapper(request, *args, **kwargs): try: user_args = parser.parse(search_args, request) except ValidationError as err: return user_error(err.messages) return view(request, *args, search_args=SearchArgs(**user_args), **kwargs)
def test_user_error(self): response = user_error("my reason") self.assertEqual(400, response.status_code) self.assertEqual('application/json', response['Content-type']) response_text = response.content.decode('utf-8') self.assertIn('my reason', response_text) json.loads(response_text) # valid json
def search(request): """Use haystack to find search results""" term = request.GET.get('q', '') version = request.GET.get('version', '') regulation = request.GET.get('regulation', '') try: page = int(request.GET.get('page', '0')) except ValueError: page = 0 if not term: return user_error('No query term') query = SearchQuerySet().models(Regulation).filter(content=term) if version: query = query.filter(version=version) if regulation: query = query.filter(regulation=regulation) start, end = page * PAGE_SIZE, (page+1) * PAGE_SIZE return success({ 'total_hits': len(query), 'results': transform_results(query[start:end]) })
def test_user_error(self): response = user_error("my reason") self.assertEqual(400, response.status_code) self.assertEqual('application/json', response['Content-type']) response_text = response.content.decode('utf-8') self.assertIn('my reason', response_text) json.loads(response_text) # valid json
def delete(request, name, doc_type, doc_id): """Delete the layer node and all of its children from the db""" params = standardize_params(doc_type, doc_id) if params.doc_type not in ('preamble', 'cfr'): return user_error('invalid doc type') storage.for_layers.bulk_delete(name, params.doc_type, params.doc_id) return success()
def add(request, name, label_id, version): """Add the layer node and all of its children to the db""" try: layer = json.loads(request.body) except ValueError: return user_error("invalid format") if not isinstance(layer, dict): return user_error("invalid format") for key in layer.keys(): # terms layer has a special attribute if not child_label_of(key, label_id) and key != "referenced": return user_error("label mismatch: %s, %s" % (label_id, key)) db.Layers().bulk_put(child_layers(name, label_id, version, layer), version, name, label_id) return success()
def add(request, name, doc_type, doc_id): """Add the layer node and all of its children to the db""" layer = request.json_body if not isinstance(layer, dict): return user_error('invalid format') params = standardize_params(doc_type, doc_id) if params.doc_type not in ('preamble', 'cfr'): return user_error('invalid doc type') for key in layer.keys(): # terms layer has a special attribute if not child_label_of(key, params.tree_id) and key != 'referenced': return user_error('label mismatch: {}, {}'.format( params.tree_id, key)) storage.for_layers.bulk_put(child_layers(params, layer), name, params.doc_type, params.doc_id) return success()
def add(request, name, doc_type, doc_id): """Add the layer node and all of its children to the db""" layer = request.json_body if not isinstance(layer, dict): return user_error('invalid format') params = standardize_params(doc_type, doc_id) if params.doc_type not in ('preamble', 'cfr'): return user_error('invalid doc type') for key in layer.keys(): # terms layer has a special attribute if not child_label_of(key, params.tree_id) and key != 'referenced': return user_error('label mismatch: {}, {}'.format( params.tree_id, key)) storage.for_layers.bulk_put(child_layers(params, layer), name, params.doc_type, params.doc_id) return success()
def add(request, docnum): """Add the notice to the db""" try: notice = anyjson.deserialize(request.body) except ValueError: return user_error('invalid format') # @todo: write a schema that verifies the notice's structure db.Notices().put(docnum, notice) return success()
def add(request, label_id, old_version, new_version): """Add the diff to the db, indexed by the label and versions""" try: diff = anyjson.deserialize(request.body) except ValueError: return user_error('invalid format') # @todo: write a schema that verifies the diff's structure db.Diffs().put(label_id, old_version, new_version, diff) return success()
def add(request, name, label_id, version): """Add the layer node and all of its children to the db""" try: layer = json.loads(request.body) except ValueError: return user_error('invalid format') if not isinstance(layer, dict): return user_error('invalid format') for key in layer.keys(): # terms layer has a special attribute if not child_label_of(key, label_id) and key != 'referenced': return user_error('label mismatch: %s, %s' % (label_id, key)) db.Layers().bulk_put(child_layers(name, label_id, version, layer), version, name, label_id) return success()
def add(request, part_or_docnum, docnum): """ Add the notice to the db """ part = part_or_docnum try: notice = anyjson.deserialize(request.body) except ValueError: return user_error('invalid format') db.Notices().put(docnum, part, notice) return success()
def add(request, part_or_docnum, docnum): """ Add the notice to the db """ part = part_or_docnum try: notice = json.loads(request.body) except ValueError: return user_error('invalid format') db.Notices().put(docnum, part, notice) return success()
def wrapper(request, *args, **kwargs): try: user_args = parser.parse(search_args, request, location="querystring") except ValidationError as err: logger.info("in val error") return user_error(err.messages) return view(request, *args, search_args=SearchArgs(**user_args), **kwargs)
def search(request, doc_type): """Use haystack to find search results""" term = request.GET.get('q', '') version = request.GET.get('version', '') regulation = request.GET.get('regulation', '') is_root = request.GET.get('is_root') is_subpart = request.GET.get('is_subpart') try: page = int(request.GET.get('page', '0')) except ValueError: page = 0 if not term: return user_error('No query term') if not validate_boolean(is_root): return user_error('Parameter "is_root" must be "true" or "false"') if not validate_boolean(is_subpart): return user_error('Parameter "is_subpart" must be "true" or "false"') query = SearchQuerySet().models(Document).filter(content=term, doc_type=doc_type) if version: query = query.filter(version=version) if regulation: query = query.filter(regulation=regulation) if is_root: query = query.filter(is_root=is_root) if is_subpart: query = query.filter(is_subpart=is_subpart) start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE return success({ 'total_hits': len(query), 'results': transform_results(query[start:end]), })
def search(request, doc_type): """Use haystack to find search results""" term = request.GET.get('q', '') version = request.GET.get('version', '') regulation = request.GET.get('regulation', '') is_root = request.GET.get('is_root') is_subpart = request.GET.get('is_subpart') try: page = int(request.GET.get('page', '0')) except ValueError: page = 0 if not term: return user_error('No query term') if not validate_boolean(is_root): return user_error('Parameter "is_root" must be "true" or "false"') if not validate_boolean(is_subpart): return user_error('Parameter "is_subpart" must be "true" or "false"') query = SearchQuerySet().models(Document).filter( content=term, doc_type=doc_type) if version: query = query.filter(version=version) if regulation: query = query.filter(regulation=regulation) if is_root: query = query.filter(is_root=is_root) if is_subpart: query = query.filter(is_subpart=is_subpart) start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE return success({ 'total_hits': len(query), 'results': transform_results(query[start:end]), })
def add(request, docnum): """Add the notice to the db""" try: notice = anyjson.deserialize(request.body) except ValueError: return user_error('invalid format') # @todo: write a schema that verifies the notice's structure cfr_parts = notice.get('cfr_parts', []) if 'cfr_part' in notice: cfr_parts.append(notice['cfr_part']) del notice['cfr_part'] notice['cfr_parts'] = cfr_parts db.Notices().put(docnum, notice) return success()
def search(request): """Search elastic search for any matches in the node's text""" term = request.GET.get('q', '') version = request.GET.get('version', '') regulation = request.GET.get('regulation', '') try: page = int(request.GET.get('page', '0')) except ValueError: page = 0 if not term: return user_error('No query term') query = { 'fields': ['text', 'label', 'version', 'regulation', 'title', 'label_string'], 'from': page * PAGE_SIZE, 'size': PAGE_SIZE, } text_match = {'match': {'text': term}} if version or regulation: term = {} if version: term['version'] = version if regulation: term['regulation'] = regulation query['query'] = { 'filtered': { 'query': text_match, 'filter': { 'term': term } } } else: query['query'] = text_match es = ElasticSearch(settings.ELASTIC_SEARCH_URLS) results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX) return success({ 'total_hits': results['hits']['total'], 'results': transform_results([h['fields'] for h in results['hits']['hits']]) })
def search(request): """Search elastic search for any matches in the node's text""" term = request.GET.get('q', '') version = request.GET.get('version', '') regulation = request.GET.get('regulation', '') try: page = int(request.GET.get('page', '0')) except ValueError: page = 0 if not term: return user_error('No query term') query = { 'fields': ['text', 'label', 'version', 'regulation', 'title', 'label_string'], 'from': page*PAGE_SIZE, 'size': PAGE_SIZE, } text_match = {'match': {'text': term}} if version or regulation: term = {} if version: term['version'] = version if regulation: term['regulation'] = regulation query['query'] = {'filtered': { 'query': text_match, 'filter': {'term': term} }} else: query['query'] = text_match es = ElasticSearch(settings.ELASTIC_SEARCH_URLS) results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX) return success({ 'total_hits': results['hits']['total'], 'results': transform_results([h['fields'] for h in results['hits']['hits']]) })
def add_all(request, part_or_docnum): """ Add the notice for all applicable CFR parts, as specified in the notice body. """ # NOTE: Be absolutely certain if you're PUTing /notice/1234-12345 # that it actually does contain content for all the parts in cfr_parts. docnum = part_or_docnum try: notice = json.loads(request.body) except ValueError: return user_error('invalid format') # This supports old-style notices that apply to multiple CFR parts. cfr_parts = notice.get('cfr_parts', []) if 'cfr_part' in notice: cfr_parts.append(notice['cfr_part']) notice['cfr_parts'] = cfr_parts for cfr_part in cfr_parts: db.Notices().put(docnum, cfr_part, notice) return success()
def add_all(request, part_or_docnum): """ Add the notice for all applicable CFR parts, as specified in the notice body. """ # NOTE: Be absolutely certain if you're PUTing /notice/1234-12345 # that it actually does contain content for all the parts in cfr_parts. docnum = part_or_docnum try: notice = json.loads(request.body) except ValueError: return user_error('invalid format') # This supports old-style notices that apply to multiple CFR parts. cfr_parts = notice.get('cfr_parts', []) if 'cfr_part' in notice: cfr_parts.append(notice['cfr_part']) notice['cfr_parts'] = cfr_parts for cfr_part in cfr_parts: db.Notices().put(docnum, cfr_part, notice) return success()
def search(request): """Use haystack to find search results""" term = request.GET.get("q", "") version = request.GET.get("version", "") regulation = request.GET.get("regulation", "") try: page = int(request.GET.get("page", "0")) except ValueError: page = 0 if not term: return user_error("No query term") query = SearchQuerySet().models(Regulation).filter(content=term) if version: query = query.filter(version=version) if regulation: query = query.filter(regulation=regulation) start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE return success({"total_hits": len(query), "results": transform_results(query[start:end])})
} } @csrf_exempt def add(request, label_id, version): """Add this regulation node and all of its children to the db""" try: node = json.loads(request.body) jsonschema.validate(node, REGULATION_SCHEMA) except ValueError: return user_error('invalid format') except jsonschema.ValidationError, e: return user_error("JSON is invalid") if label_id != '-'.join(node['label']): return user_error('label mismatch') to_save = [] def add_node(node): node = dict(node) # copy to_save.append(node) for child in node['children']: add_node(child) add_node(node) db.Regulations().bulk_put(to_save, version, label_id) return success()
def wrapped(request, *args, **kwargs): try: request.json_body = json.loads(request.body.decode('utf-8')) return func(request, *args, **kwargs) except (ValueError, UnicodeError): return user_error('invalid format')
def test_user_error(self): response = user_error("my reason") self.assertEqual(400, response.status_code) self.assertEqual('application/json', response['Content-type']) self.assertTrue('my reason' in response.content) json.loads(response.content) # valid json
} } @csrf_exempt def add(request, label_id, version): """Add this regulation node and all of its children to the db""" try: node = anyjson.deserialize(request.body) jsonschema.validate(node, REGULATION_SCHEMA) except ValueError: return user_error('invalid format') except jsonschema.ValidationError, e: return user_error("JSON is invalid") if label_id != '-'.join(node['label']): return user_error('label mismatch') to_save = [] def add_node(node): node = dict(node) # copy to_save.append(node) for child in node['children']: add_node(child) add_node(node) db.Regulations().bulk_put(to_save, version, label_id) return success()