def get(request, name, label_id, version):
"""Find and return the layer with this name + version + label"""
layer = db.Layers().get(name, label_id, version)
if layer is not None:
return success(layer)
else:
return four_oh_four()
def test_success_full(self):
structure = {'some': {'structure': 1}}
response = success(structure)
self.assertEqual(200, response.status_code)
self.assertEqual('application/json', response['Content-type'])
self.assertEqual(structure,
json.loads(response.content.decode('utf-8')))
def listing(request, doc_type, label_id=None):
"""List versions of the requested (label_id) regulation; or all regulations
if label_id is None"""
if label_id:
reg_versions = storage.for_documents.listing(doc_type, label_id)
notices = storage.for_notices.listing(label_id.split('-')[0])
else:
reg_versions = storage.for_documents.listing(doc_type)
notices = storage.for_notices.listing()
by_date = defaultdict(list)
for notice in (n for n in notices if 'effective_on' in n):
by_date[notice['effective_on']].append(notice)
regs = []
for effective_date in sorted(by_date.keys(), reverse=True):
notices = [(n['document_number'], n['effective_on'])
for n in by_date[effective_date]]
notices = sorted(notices, reverse=True)
found_latest = set()
for doc_number, date in notices:
for version, reg_part in reg_versions:
if doc_number == version and reg_part in found_latest:
regs.append({'version': version, 'regulation': reg_part})
elif doc_number == version:
found_latest.add(reg_part)
regs.append({'version': version, 'by_date': date,
'regulation': reg_part})
if regs:
return success({'versions': regs})
else:
return four_oh_four()
def search(request):
"""Use haystack to find search results"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = SearchQuerySet().models(Regulation).filter(content=term)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE
return success({
'total_hits': len(query),
'results': transform_results(query[start:end])
})
def get(request, docnum):
"""Find and return the notice with this docnum"""
notice = db.Notices().get(docnum)
if notice:
return success(notice)
else:
return four_oh_four()
def listing(request, label_id=None):
"""List versions of the requested (label_id) regulation; or all regulations
if label_id is None"""
if label_id:
reg_versions = db.Regulations().listing(label_id)
notices = db.Notices().listing(label_id.split("-")[0])
else:
reg_versions = db.Regulations().listing()
notices = db.Notices().listing()
by_date = defaultdict(list)
for notice in (n for n in notices if "effective_on" in n):
by_date[notice["effective_on"]].append(notice)
regs = []
for effective_date in sorted(by_date.keys(), reverse=True):
notices = [(n["document_number"], n["effective_on"]) for n in by_date[effective_date]]
notices = sorted(notices, reverse=True)
found_latest = set()
for doc_number, date in notices:
for version, reg_part in reg_versions:
if doc_number == version and reg_part in found_latest:
regs.append({"version": version, "regulation": reg_part})
elif doc_number == version:
found_latest.add(reg_part)
regs.append({"version": version, "by_date": date, "regulation": reg_part})
if regs:
return success({"versions": regs})
else:
return four_oh_four()
def search(request):
"""Use haystack to find search results"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = SearchQuerySet().models(Regulation).filter(content=term)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
start, end = page * PAGE_SIZE, (page+1) * PAGE_SIZE
return success({
'total_hits': len(query),
'results': transform_results(query[start:end])
})
def get(request, doc_type, label_id, version=None):
"""Find and return the regulation with this version and label"""
regulation = storage.for_documents.get(doc_type, label_id, version)
if regulation is not None:
return success(regulation)
else:
return four_oh_four()
def get(request, label_id, old_version, new_version):
"""Find and return the diff with the provided label / versions"""
diff = storage.for_diffs.get(label_id, old_version, new_version)
if diff is not None:
return success(diff)
else:
return four_oh_four()
def search(request, doc_type, search_args):
"""Search elastic search for any matches in the node's text"""
query = {
'fields': ['text', 'label', 'version', 'regulation', 'title',
'label_string'],
'from': search_args.page * search_args.page_size,
'size': search_args.page_size,
}
text_match = {'match': {'text': search_args.q, 'doc_type': doc_type}}
if search_args.version or search_args.regulation:
term = {}
if search_args.version:
term['version'] = search_args.version
if search_args.regulation:
term['regulation'] = search_args.regulation
if search_args.is_root is not None:
term['is_root'] = search_args.is_root
if search_args.is_subpart is not None:
term['is_subpart'] = search_args.is_subpart
query['query'] = {'filtered': {
'query': text_match,
'filter': {'term': term}
}}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits': results['hits']['total'],
'results': transform_results([h['fields'] for h in
results['hits']['hits']])
})
def add(request, label_id, version):
"""Add this regulation node and all of its children to the db"""
try:
node = json.loads(request.body)
jsonschema.validate(node, REGULATION_SCHEMA)
except ValueError:
return user_error('invalid format')
except jsonschema.ValidationError:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
to_save = []
labels_seen = set()
def add_node(node):
label_tuple = tuple(node['label'])
if label_tuple in labels_seen:
logging.warning("Repeat label: %s", label_tuple)
labels_seen.add(label_tuple)
node = dict(node) # copy
to_save.append(node)
for child in node['children']:
add_node(child)
add_node(node)
db.Regulations().bulk_put(to_save, version, label_id)
return success()
def listing(request, label_id):
"""List versions of this regulation"""
part = label_id.split('-')[0]
notices = db.Notices().listing(label_id)
by_date = defaultdict(list)
for notice in (n for n in notices if 'effective_on' in n):
by_date[notice['effective_on']].append(notice)
reg_versions = set(db.Regulations().listing(label_id))
regs = []
for effective_date in sorted(by_date.keys(), reverse=True):
notices = [(n['document_number'], n['effective_on'])
for n in by_date[effective_date]]
notices = sorted(notices, reverse=True)
found_latest = False
for version, effective in ((v, d) for v, d in notices
if v in reg_versions):
if found_latest:
regs.append({'version': version})
else:
found_latest = True
regs.append({'version': version, 'by_date': effective})
if regs:
return success({'versions': regs})
else:
return four_oh_four()
def get(request, docnum):
"""Find and return the notice with this docnum"""
notice = storage.for_notices.get(docnum)
if notice is not None:
return success(notice)
else:
return four_oh_four()
def get(request, docnum):
"""Find and return the notice with this docnum"""
notice = db.Notices().get(docnum)
if notice:
return success(notice)
else:
return four_oh_four()
def get(request, label_id, version):
"""Find and return the regulation with this version and label"""
regulation = db.Regulations().get(label_id, version)
if regulation:
return success(regulation)
else:
return four_oh_four()
def get(request, label_id, old_version, new_version):
"""Find and return the diff with the provided label / versions"""
diff = db.Diffs().get(label_id, old_version, new_version)
if diff:
return success(diff)
else:
return four_oh_four()
def get(request, label_id, version):
"""Find and return the regulation with this version and label"""
regulation = db.Regulations().get(label_id, version)
if regulation:
return success(regulation)
else:
return four_oh_four()
def add(request, label_id, old_version, new_version):
"""Add the diff to the db, indexed by the label and versions"""
# @todo: write a schema that verifies the diff's structure
storage.for_diffs.delete(label_id, old_version, new_version)
storage.for_diffs.insert(
label_id, old_version, new_version, request.json_body)
return success()
def get(request, name, doc_type, doc_id):
"""Find and return the layer with this name, referring to this doc_id"""
params = standardize_params(doc_type, doc_id)
layer = storage.for_layers.get(name, params.doc_type, params.doc_id)
if layer is not None:
return success(layer)
else:
return four_oh_four()
def get(request, part_or_docnum, docnum):
""" Find and return the notice with this docnum and part """
part = part_or_docnum
notice = db.Notices().get(doc_number=docnum, part=part)
if notice:
return success(notice)
else:
return four_oh_four()
def get(request, part_or_docnum, docnum):
""" Find and return the notice with this docnum and part """
part = part_or_docnum
notice = db.Notices().get(doc_number=docnum, part=part)
if notice:
return success(notice)
else:
return four_oh_four()
def delete(request, name, doc_type, doc_id):
"""Delete the layer node and all of its children from the db"""
params = standardize_params(doc_type, doc_id)
if params.doc_type not in ('preamble', 'cfr'):
return user_error('invalid doc type')
storage.for_layers.bulk_delete(name, params.doc_type, params.doc_id)
return success()
def search(request, doc_type):
"""Search elastic search for any matches in the node's text"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
is_root = request.GET.get('is_root')
is_subpart = request.GET.get('is_subpart')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
if not validate_boolean(is_root):
return user_error('Parameter "is_root" must be "true" or "false"')
if not validate_boolean(is_subpart):
return user_error('Parameter "is_subpart" must be "true" or "false"')
query = {
'fields':
['text', 'label', 'version', 'regulation', 'title', 'label_string'],
'from':
page * PAGE_SIZE,
'size':
PAGE_SIZE,
}
text_match = {'match': {'text': term, 'doc_type': doc_type}}
if version or regulation:
term = {}
if version:
term['version'] = version
if regulation:
term['regulation'] = regulation
if is_root:
term['is_root'] = is_root
if is_subpart:
term['is_subpart'] = is_subpart
query['query'] = {
'filtered': {
'query': text_match,
'filter': {
'term': term
}
}
}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits':
results['hits']['total'],
'results':
transform_results([h['fields'] for h in results['hits']['hits']])
})
def search(request, doc_type, search_args):
sections = matching_sections(search_args)
start = search_args.page * search_args.page_size
end = start + search_args.page_size
return success({
'total_hits': sections.count(),
'results': transform_results(sections[start:end], search_args.q),
})
def add(request, label_id, old_version, new_version):
"""Add the diff to the db, indexed by the label and versions"""
try:
diff = anyjson.deserialize(request.body)
except ValueError:
return user_error('invalid format')
# @todo: write a schema that verifies the diff's structure
db.Diffs().put(label_id, old_version, new_version, diff)
return success()
def add(request, docnum):
"""Add the notice to the db"""
try:
notice = anyjson.deserialize(request.body)
except ValueError:
return user_error('invalid format')
# @todo: write a schema that verifies the notice's structure
db.Notices().put(docnum, notice)
return success()
def add(request, part_or_docnum, docnum):
""" Add the notice to the db """
part = part_or_docnum
try:
notice = json.loads(request.body)
except ValueError:
return user_error('invalid format')
db.Notices().put(docnum, part, notice)
return success()
def add(request, part_or_docnum, docnum):
""" Add the notice to the db """
part = part_or_docnum
try:
notice = anyjson.deserialize(request.body)
except ValueError:
return user_error('invalid format')
db.Notices().put(docnum, part, notice)
return success()
def add(request, doc_type, label_id, version=None):
"""Add this document node and all of its children to the db"""
try:
node = request.json_body
jsonschema.validate(node, REGULATION_SCHEMA)
except jsonschema.ValidationError:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
write_node(node, doc_type, label_id, version)
return success()
def add(request, docnum):
"""Add the notice to the db"""
notice = request.json_body
# @todo: write a schema that verifies the notice's structure
cfr_parts = notice.get('cfr_parts', [])
if 'cfr_part' in notice:
cfr_parts.append(notice['cfr_part'])
del notice['cfr_part']
notice['cfr_parts'] = cfr_parts
storage.for_notices.put(docnum, notice)
return success()
def add(request, doc_type, label_id, version=None):
"""Add this document node and all of its children to the db"""
try:
node = request.json_body
jsonschema.validate(node, REGULATION_SCHEMA)
except jsonschema.ValidationError:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
write_node(node, doc_type, label_id, version)
return success()
def search(request, doc_type):
"""Search elastic search for any matches in the node's text"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
is_root = request.GET.get('is_root')
is_subpart = request.GET.get('is_subpart')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
if not validate_boolean(is_root):
return user_error('Parameter "is_root" must be "true" or "false"')
if not validate_boolean(is_subpart):
return user_error('Parameter "is_subpart" must be "true" or "false"')
query = {
'fields': ['text', 'label', 'version', 'regulation', 'title',
'label_string'],
'from': page * PAGE_SIZE,
'size': PAGE_SIZE,
}
text_match = {'match': {'text': term, 'doc_type': doc_type}}
if version or regulation:
term = {}
if version:
term['version'] = version
if regulation:
term['regulation'] = regulation
if is_root:
term['is_root'] = is_root
if is_subpart:
term['is_subpart'] = is_subpart
query['query'] = {'filtered': {
'query': text_match,
'filter': {'term': term}
}}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits': results['hits']['total'],
'results': transform_results([h['fields'] for h in
results['hits']['hits']])
})
def add(request, docnum):
"""Add the notice to the db"""
try:
notice = anyjson.deserialize(request.body)
except ValueError:
return user_error('invalid format')
# @todo: write a schema that verifies the notice's structure
cfr_parts = notice.get('cfr_parts', [])
if 'cfr_part' in notice:
cfr_parts.append(notice['cfr_part'])
del notice['cfr_part']
notice['cfr_parts'] = cfr_parts
db.Notices().put(docnum, notice)
return success()
def search(request):
"""Search elastic search for any matches in the node's text"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = {
'fields':
['text', 'label', 'version', 'regulation', 'title', 'label_string'],
'from':
page * PAGE_SIZE,
'size':
PAGE_SIZE,
}
text_match = {'match': {'text': term}}
if version or regulation:
term = {}
if version:
term['version'] = version
if regulation:
term['regulation'] = regulation
query['query'] = {
'filtered': {
'query': text_match,
'filter': {
'term': term
}
}
}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits':
results['hits']['total'],
'results':
transform_results([h['fields'] for h in results['hits']['hits']])
})
def add(request, name, label_id, version):
"""Add the layer node and all of its children to the db"""
try:
layer = json.loads(request.body)
except ValueError:
return user_error("invalid format")
if not isinstance(layer, dict):
return user_error("invalid format")
for key in layer.keys():
# terms layer has a special attribute
if not child_label_of(key, label_id) and key != "referenced":
return user_error("label mismatch: %s, %s" % (label_id, key))
db.Layers().bulk_put(child_layers(name, label_id, version, layer), version, name, label_id)
return success()
def add(request, name, label_id, version):
"""Add the layer node and all of its children to the db"""
try:
layer = json.loads(request.body)
except ValueError:
return user_error('invalid format')
if not isinstance(layer, dict):
return user_error('invalid format')
for key in layer.keys():
# terms layer has a special attribute
if not child_label_of(key, label_id) and key != 'referenced':
return user_error('label mismatch: %s, %s' % (label_id, key))
db.Layers().bulk_put(child_layers(name, label_id, version, layer),
version, name, label_id)
return success()
def add(request, name, doc_type, doc_id):
"""Add the layer node and all of its children to the db"""
layer = request.json_body
if not isinstance(layer, dict):
return user_error('invalid format')
params = standardize_params(doc_type, doc_id)
if params.doc_type not in ('preamble', 'cfr'):
return user_error('invalid doc type')
for key in layer.keys():
# terms layer has a special attribute
if not child_label_of(key, params.tree_id) and key != 'referenced':
return user_error('label mismatch: {}, {}'.format(
params.tree_id, key))
storage.for_layers.bulk_put(child_layers(params, layer), name,
params.doc_type, params.doc_id)
return success()
def add(request, name, doc_type, doc_id):
"""Add the layer node and all of its children to the db"""
layer = request.json_body
if not isinstance(layer, dict):
return user_error('invalid format')
params = standardize_params(doc_type, doc_id)
if params.doc_type not in ('preamble', 'cfr'):
return user_error('invalid doc type')
for key in layer.keys():
# terms layer has a special attribute
if not child_label_of(key, params.tree_id) and key != 'referenced':
return user_error('label mismatch: {}, {}'.format(
params.tree_id, key))
storage.for_layers.bulk_put(child_layers(params, layer), name,
params.doc_type, params.doc_id)
return success()
def search(request, doc_type, search_args):
"""Use haystack to find search results"""
query = SearchQuerySet().models(Document).filter(
content=search_args.q, doc_type=doc_type)
if search_args.version:
query = query.filter(version=search_args.version)
if search_args.regulation:
query = query.filter(regulation=search_args.regulation)
if search_args.is_root is not None:
query = query.filter(is_root=search_args.is_root)
if search_args.is_subpart is not None:
query = query.filter(is_subpart=search_args.is_subpart)
start = search_args.page * search_args.page_size
end = start + search_args.page_size
return success({
'total_hits': len(query),
'results': transform_results(query[start:end]),
})
def search(request, doc_type, search_args):
"""Use haystack to find search results"""
query = SearchQuerySet().models(Document).filter(content=search_args.q,
doc_type=doc_type)
if search_args.version:
query = query.filter(version=search_args.version)
if search_args.regulation:
query = query.filter(regulation=search_args.regulation)
if search_args.is_root is not None:
query = query.filter(is_root=search_args.is_root)
if search_args.is_subpart is not None:
query = query.filter(is_subpart=search_args.is_subpart)
start = search_args.page * search_args.page_size
end = start + search_args.page_size
return success({
'total_hits': len(query),
'results': transform_results(query[start:end]),
})
def search(request):
"""Search elastic search for any matches in the node's text"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = {
'fields': ['text', 'label', 'version', 'regulation', 'title',
'label_string'],
'from': page*PAGE_SIZE,
'size': PAGE_SIZE,
}
text_match = {'match': {'text': term}}
if version or regulation:
term = {}
if version:
term['version'] = version
if regulation:
term['regulation'] = regulation
query['query'] = {'filtered': {
'query': text_match,
'filter': {'term': term}
}}
else:
query['query'] = text_match
es = ElasticSearch(settings.ELASTIC_SEARCH_URLS)
results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX)
return success({
'total_hits': results['hits']['total'],
'results': transform_results([h['fields'] for h in
results['hits']['hits']])
})
def add_all(request, part_or_docnum):
""" Add the notice for all applicable CFR parts, as specified in the
notice body. """
# NOTE: Be absolutely certain if you're PUTing /notice/1234-12345
# that it actually does contain content for all the parts in cfr_parts.
docnum = part_or_docnum
try:
notice = json.loads(request.body)
except ValueError:
return user_error('invalid format')
# This supports old-style notices that apply to multiple CFR parts.
cfr_parts = notice.get('cfr_parts', [])
if 'cfr_part' in notice:
cfr_parts.append(notice['cfr_part'])
notice['cfr_parts'] = cfr_parts
for cfr_part in cfr_parts:
db.Notices().put(docnum, cfr_part, notice)
return success()
def listing(request, doc_type, label_id=None):
"""List versions of the requested (label_id) regulation; or all regulations
if label_id is None"""
if label_id:
reg_versions = storage.for_documents.listing(doc_type, label_id)
notices = storage.for_notices.listing(label_id.split('-')[0])
else:
reg_versions = storage.for_documents.listing(doc_type)
notices = storage.for_notices.listing()
by_date = defaultdict(list)
for notice in (n for n in notices if 'effective_on' in n):
by_date[notice['effective_on']].append(notice)
regs = []
for effective_date in sorted(by_date.keys(), reverse=True):
notices = [(n['document_number'], n['effective_on'])
for n in by_date[effective_date]]
notices = sorted(notices, reverse=True)
found_latest = set()
for doc_number, date in notices:
for version, reg_part in reg_versions:
if doc_number == version and reg_part in found_latest:
regs.append({'version': version, 'regulation': reg_part})
elif doc_number == version:
found_latest.add(reg_part)
regs.append({
'version': version,
'by_date': date,
'regulation': reg_part
})
if regs:
return success({'versions': regs})
else:
return four_oh_four()
def search(request, doc_type):
"""Use haystack to find search results"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
is_root = request.GET.get('is_root')
is_subpart = request.GET.get('is_subpart')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
if not validate_boolean(is_root):
return user_error('Parameter "is_root" must be "true" or "false"')
if not validate_boolean(is_subpart):
return user_error('Parameter "is_subpart" must be "true" or "false"')
query = SearchQuerySet().models(Document).filter(content=term,
doc_type=doc_type)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
if is_root:
query = query.filter(is_root=is_root)
if is_subpart:
query = query.filter(is_subpart=is_subpart)
start, end = page * PAGE_SIZE, (page + 1) * PAGE_SIZE
return success({
'total_hits': len(query),
'results': transform_results(query[start:end]),
})
def delete(request, doc_type, label_id, version=None):
"""Delete this document node and all of its children from the db"""
storage.for_documents.bulk_delete(doc_type, label_id, version)
return success()
def test_success_empty(self):
response = success()
self.assertEqual(204, response.status_code)
self.assertEqual(0, len(response.content))
def delete(request, docnum):
"""Delete the notice from the db"""
storage.for_notices.delete(docnum)
return success()
}
}
@csrf_exempt
def add(request, label_id, version):
"""Add this regulation node and all of its children to the db"""
try:
node = json.loads(request.body)
jsonschema.validate(node, REGULATION_SCHEMA)
except ValueError:
return user_error('invalid format')
except jsonschema.ValidationError, e:
return user_error("JSON is invalid")
if label_id != '-'.join(node['label']):
return user_error('label mismatch')
to_save = []
def add_node(node):
node = dict(node) # copy
to_save.append(node)
for child in node['children']:
add_node(child)
add_node(node)
db.Regulations().bulk_put(to_save, version, label_id)
return success()
def listing(request):
"""Find and return all notices"""
return success({
'results':
storage.for_notices.listing(request.GET.get('part', None))
})
def listing(request):
"""Find and return all notices"""
return success(
{'results': db.Notices().listing(request.GET.get('part', None))})
def listing(request, part_or_docnum=None):
"""Find and return all notices"""
part = part_or_docnum
return success({'results': db.Notices().listing(part=part)})
