def add(request, label_id, version):
"""Add this regulation node and all of its children to the db"""
try:
node = json.loads(request.body)
jsonschema.validate(node, REGULATION_SCHEMA)
except ValueError:
return user_error('invalid format')
except jsonschema.ValidationError:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
to_save = []
labels_seen = set()
def add_node(node):
label_tuple = tuple(node['label'])
if label_tuple in labels_seen:
logging.warning("Repeat label: %s", label_tuple)
labels_seen.add(label_tuple)
node = dict(node) # copy
to_save.append(node)
for child in node['children']:
add_node(child)
add_node(node)
db.Regulations().bulk_put(to_save, version, label_id)
return success()
def search(request, doc_type):
"""Search elastic search for any matches in the node's text"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
is_root = request.GET.get('is_root')
is_subpart = request.GET.get('is_subpart')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
if not validate_boolean(is_root):
return user_error('Parameter "is_root" must be "true" or "false"')
if not validate_boolean(is_subpart):
return user_error('Parameter "is_subpart" must be "true" or "false"')
query = {
'fields':
['text', 'label', 'version', 'regulation', 'title', 'label_string'],
'from':
page * PAGE_SIZE,
'size':
PAGE_SIZE,
}
text_match = {'match': {'text': term, 'doc_type': doc_type}}
if version or regulation:
term = {}
if version:
term['version'] = version
if regulation:
term['regulation'] = regulation
if is_root:
term['is_root'] = is_root
if is_subpart:
term['is_subpart'] = is_subpart
query['query'] = {
'filtered': {
'query': text_match,
'filter': {
'term': term
}
}
}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits':
results['hits']['total'],
'results':
transform_results([h['fields'] for h in results['hits']['hits']])
})
def add(request, label_id, version):
"""Add this regulation node and all of its children to the db"""
try:
node = json.loads(request.body)
jsonschema.validate(node, REGULATION_SCHEMA)
except ValueError:
return user_error('invalid format')
except jsonschema.ValidationError, e:
return user_error("JSON is invalid")
def add(request, label_id, version):
"""Add this regulation node and all of its children to the db"""
try:
node = anyjson.deserialize(request.body)
jsonschema.validate(node, REGULATION_SCHEMA)
except ValueError:
return user_error('invalid format')
except jsonschema.ValidationError, e:
return user_error("JSON is invalid")
def add(request, doc_type, label_id, version=None):
"""Add this document node and all of its children to the db"""
try:
node = request.json_body
jsonschema.validate(node, REGULATION_SCHEMA)
except jsonschema.ValidationError:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
write_node(node, doc_type, label_id, version)
return success()
def add(request, doc_type, label_id, version=None):
"""Add this document node and all of its children to the db"""
try:
node = request.json_body
jsonschema.validate(node, REGULATION_SCHEMA)
except jsonschema.ValidationError:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
write_node(node, doc_type, label_id, version)
return success()
def search(request, doc_type):
"""Search elastic search for any matches in the node's text"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
is_root = request.GET.get('is_root')
is_subpart = request.GET.get('is_subpart')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
if not validate_boolean(is_root):
return user_error('Parameter "is_root" must be "true" or "false"')
if not validate_boolean(is_subpart):
return user_error('Parameter "is_subpart" must be "true" or "false"')
query = {
'fields': ['text', 'label', 'version', 'regulation', 'title',
'label_string'],
'from': page * PAGE_SIZE,
'size': PAGE_SIZE,
}
text_match = {'match': {'text': term, 'doc_type': doc_type}}
if version or regulation:
term = {}
if version:
term['version'] = version
if regulation:
term['regulation'] = regulation
if is_root:
term['is_root'] = is_root
if is_subpart:
term['is_subpart'] = is_subpart
query['query'] = {'filtered': {
'query': text_match,
'filter': {'term': term}
}}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits': results['hits']['total'],
'results': transform_results([h['fields'] for h in
results['hits']['hits']])
})
def search(request):
"""Use haystack to find search results"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = SearchQuerySet().models(Regulation).filter(content=term)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE
return success({
'total_hits': len(query),
'results': transform_results(query[start:end])
})
def wrapper(request, *args, **kwargs):
try:
user_args = parser.parse(search_args, request)
except ValidationError as err:
return user_error(err.messages)
return view(request, *args, search_args=SearchArgs(**user_args),
**kwargs)
def test_user_error(self):
response = user_error("my reason")
self.assertEqual(400, response.status_code)
self.assertEqual('application/json', response['Content-type'])
response_text = response.content.decode('utf-8')
self.assertIn('my reason', response_text)
json.loads(response_text) # valid json
def search(request):
"""Use haystack to find search results"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = SearchQuerySet().models(Regulation).filter(content=term)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
start, end = page * PAGE_SIZE, (page+1) * PAGE_SIZE
return success({
'total_hits': len(query),
'results': transform_results(query[start:end])
})
def test_user_error(self):
response = user_error("my reason")
self.assertEqual(400, response.status_code)
self.assertEqual('application/json', response['Content-type'])
response_text = response.content.decode('utf-8')
self.assertIn('my reason', response_text)
json.loads(response_text) # valid json
def delete(request, name, doc_type, doc_id):
"""Delete the layer node and all of its children from the db"""
params = standardize_params(doc_type, doc_id)
if params.doc_type not in ('preamble', 'cfr'):
return user_error('invalid doc type')
storage.for_layers.bulk_delete(name, params.doc_type, params.doc_id)
return success()
def add(request, name, label_id, version):
"""Add the layer node and all of its children to the db"""
try:
layer = json.loads(request.body)
except ValueError:
return user_error("invalid format")
if not isinstance(layer, dict):
return user_error("invalid format")
for key in layer.keys():
# terms layer has a special attribute
if not child_label_of(key, label_id) and key != "referenced":
return user_error("label mismatch: %s, %s" % (label_id, key))
db.Layers().bulk_put(child_layers(name, label_id, version, layer), version, name, label_id)
return success()
def add(request, name, doc_type, doc_id):
"""Add the layer node and all of its children to the db"""
layer = request.json_body
if not isinstance(layer, dict):
return user_error('invalid format')
params = standardize_params(doc_type, doc_id)
if params.doc_type not in ('preamble', 'cfr'):
return user_error('invalid doc type')
for key in layer.keys():
# terms layer has a special attribute
if not child_label_of(key, params.tree_id) and key != 'referenced':
return user_error('label mismatch: {}, {}'.format(
params.tree_id, key))
storage.for_layers.bulk_put(child_layers(params, layer), name,
params.doc_type, params.doc_id)
return success()
def add(request, name, doc_type, doc_id):
"""Add the layer node and all of its children to the db"""
layer = request.json_body
if not isinstance(layer, dict):
return user_error('invalid format')
params = standardize_params(doc_type, doc_id)
if params.doc_type not in ('preamble', 'cfr'):
return user_error('invalid doc type')
for key in layer.keys():
# terms layer has a special attribute
if not child_label_of(key, params.tree_id) and key != 'referenced':
return user_error('label mismatch: {}, {}'.format(
params.tree_id, key))
storage.for_layers.bulk_put(child_layers(params, layer), name,
params.doc_type, params.doc_id)
return success()
def add(request, docnum):
"""Add the notice to the db"""
try:
notice = anyjson.deserialize(request.body)
except ValueError:
return user_error('invalid format')
# @todo: write a schema that verifies the notice's structure
db.Notices().put(docnum, notice)
return success()
def add(request, label_id, old_version, new_version):
"""Add the diff to the db, indexed by the label and versions"""
try:
diff = anyjson.deserialize(request.body)
except ValueError:
return user_error('invalid format')
# @todo: write a schema that verifies the diff's structure
db.Diffs().put(label_id, old_version, new_version, diff)
return success()
def add(request, name, label_id, version):
"""Add the layer node and all of its children to the db"""
try:
layer = json.loads(request.body)
except ValueError:
return user_error('invalid format')
if not isinstance(layer, dict):
return user_error('invalid format')
for key in layer.keys():
# terms layer has a special attribute
if not child_label_of(key, label_id) and key != 'referenced':
return user_error('label mismatch: %s, %s' % (label_id, key))
db.Layers().bulk_put(child_layers(name, label_id, version, layer),
version, name, label_id)
return success()
def add(request, part_or_docnum, docnum):
""" Add the notice to the db """
part = part_or_docnum
try:
notice = anyjson.deserialize(request.body)
except ValueError:
return user_error('invalid format')
db.Notices().put(docnum, part, notice)
return success()
def add(request, part_or_docnum, docnum):
""" Add the notice to the db """
part = part_or_docnum
try:
notice = json.loads(request.body)
except ValueError:
return user_error('invalid format')
db.Notices().put(docnum, part, notice)
return success()
def wrapper(request, *args, **kwargs):
try:
user_args = parser.parse(search_args,
request,
location="querystring")
except ValidationError as err:
logger.info("in val error")
return user_error(err.messages)
return view(request,
*args,
search_args=SearchArgs(**user_args),
**kwargs)
def search(request, doc_type):
"""Use haystack to find search results"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
is_root = request.GET.get('is_root')
is_subpart = request.GET.get('is_subpart')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
if not validate_boolean(is_root):
return user_error('Parameter "is_root" must be "true" or "false"')
if not validate_boolean(is_subpart):
return user_error('Parameter "is_subpart" must be "true" or "false"')
query = SearchQuerySet().models(Document).filter(content=term,
doc_type=doc_type)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
if is_root:
query = query.filter(is_root=is_root)
if is_subpart:
query = query.filter(is_subpart=is_subpart)
start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE
return success({
'total_hits': len(query),
'results': transform_results(query[start:end]),
})
def search(request, doc_type):
"""Use haystack to find search results"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
is_root = request.GET.get('is_root')
is_subpart = request.GET.get('is_subpart')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
if not validate_boolean(is_root):
return user_error('Parameter "is_root" must be "true" or "false"')
if not validate_boolean(is_subpart):
return user_error('Parameter "is_subpart" must be "true" or "false"')
query = SearchQuerySet().models(Document).filter(
content=term, doc_type=doc_type)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
if is_root:
query = query.filter(is_root=is_root)
if is_subpart:
query = query.filter(is_subpart=is_subpart)
start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE
return success({
'total_hits': len(query),
'results': transform_results(query[start:end]),
})
def add(request, docnum):
"""Add the notice to the db"""
try:
notice = anyjson.deserialize(request.body)
except ValueError:
return user_error('invalid format')
# @todo: write a schema that verifies the notice's structure
cfr_parts = notice.get('cfr_parts', [])
if 'cfr_part' in notice:
cfr_parts.append(notice['cfr_part'])
del notice['cfr_part']
notice['cfr_parts'] = cfr_parts
db.Notices().put(docnum, notice)
return success()
def search(request):
"""Search elastic search for any matches in the node's text"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = {
'fields':
['text', 'label', 'version', 'regulation', 'title', 'label_string'],
'from':
page * PAGE_SIZE,
'size':
PAGE_SIZE,
}
text_match = {'match': {'text': term}}
if version or regulation:
term = {}
if version:
term['version'] = version
if regulation:
term['regulation'] = regulation
query['query'] = {
'filtered': {
'query': text_match,
'filter': {
'term': term
}
}
}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits':
results['hits']['total'],
'results':
transform_results([h['fields'] for h in results['hits']['hits']])
})
def search(request):
"""Search elastic search for any matches in the node's text"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = {
'fields': ['text', 'label', 'version', 'regulation', 'title',
'label_string'],
'from': page*PAGE_SIZE,
'size': PAGE_SIZE,
}
text_match = {'match': {'text': term}}
if version or regulation:
term = {}
if version:
term['version'] = version
if regulation:
term['regulation'] = regulation
query['query'] = {'filtered': {
'query': text_match,
'filter': {'term': term}
}}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits': results['hits']['total'],
'results': transform_results([h['fields'] for h in
results['hits']['hits']])
})
def add_all(request, part_or_docnum):
""" Add the notice for all applicable CFR parts, as specified in the
notice body. """
# NOTE: Be absolutely certain if you're PUTing /notice/1234-12345
# that it actually does contain content for all the parts in cfr_parts.
docnum = part_or_docnum
try:
notice = json.loads(request.body)
except ValueError:
return user_error('invalid format')
# This supports old-style notices that apply to multiple CFR parts.
cfr_parts = notice.get('cfr_parts', [])
if 'cfr_part' in notice:
cfr_parts.append(notice['cfr_part'])
notice['cfr_parts'] = cfr_parts
for cfr_part in cfr_parts:
db.Notices().put(docnum, cfr_part, notice)
return success()
def add_all(request, part_or_docnum):
""" Add the notice for all applicable CFR parts, as specified in the
notice body. """
# NOTE: Be absolutely certain if you're PUTing /notice/1234-12345
# that it actually does contain content for all the parts in cfr_parts.
docnum = part_or_docnum
try:
notice = json.loads(request.body)
except ValueError:
return user_error('invalid format')
# This supports old-style notices that apply to multiple CFR parts.
cfr_parts = notice.get('cfr_parts', [])
if 'cfr_part' in notice:
cfr_parts.append(notice['cfr_part'])
notice['cfr_parts'] = cfr_parts
for cfr_part in cfr_parts:
db.Notices().put(docnum, cfr_part, notice)
return success()
def search(request):
"""Use haystack to find search results"""
term = request.GET.get("q", "")
version = request.GET.get("version", "")
regulation = request.GET.get("regulation", "")
try:
page = int(request.GET.get("page", "0"))
except ValueError:
page = 0
if not term:
return user_error("No query term")
query = SearchQuerySet().models(Regulation).filter(content=term)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE
return success({"total_hits": len(query), "results": transform_results(query[start:end])})
}
}
@csrf_exempt
def add(request, label_id, version):
"""Add this regulation node and all of its children to the db"""
try:
node = json.loads(request.body)
jsonschema.validate(node, REGULATION_SCHEMA)
except ValueError:
return user_error('invalid format')
except jsonschema.ValidationError, e:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
to_save = []
def add_node(node):
node = dict(node) # copy
to_save.append(node)
for child in node['children']:
add_node(child)
add_node(node)
db.Regulations().bulk_put(to_save, version, label_id)
return success()
def wrapped(request, *args, **kwargs):
try:
request.json_body = json.loads(request.body.decode('utf-8'))
return func(request, *args, **kwargs)
except (ValueError, UnicodeError):
return user_error('invalid format')
def test_user_error(self):
response = user_error("my reason")
self.assertEqual(400, response.status_code)
self.assertEqual('application/json', response['Content-type'])
self.assertTrue('my reason' in response.content)
json.loads(response.content) # valid json
}
}
@csrf_exempt
def add(request, label_id, version):
"""Add this regulation node and all of its children to the db"""
try:
node = anyjson.deserialize(request.body)
jsonschema.validate(node, REGULATION_SCHEMA)
except ValueError:
return user_error('invalid format')
except jsonschema.ValidationError, e:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
to_save = []
def add_node(node):
node = dict(node) # copy
to_save.append(node)
for child in node['children']:
add_node(child)
add_node(node)
db.Regulations().bulk_put(to_save, version, label_id)
return success()
