def import_mjson():
logging.getLogger('storage').setLevel(logging.INFO)
def batched(iterator, count=10):
batch = []
for value in iterator:
batch.append(value)
if len(batch) >= count:
yield batch
batch = []
if batch:
yield batch
def read_json_lines(stream):
for line in stream:
yield flask.json.loads(line)
def load_document(data):
doc = naturasites.schema.SpaDoc(data)
assert doc.validate(
), '%s does not validate' % data['section1']['code']
assert doc.value == data, 'failed round-tripping the json data'
return doc
db = get_db(create_app())
for batch in batched(load_document(d) for d in read_json_lines(sys.stdin)):
db.save_document_batch(batch)
sys.stdout.write('.')
sys.stdout.flush()
print ' done'
def import_mjson():
logging.getLogger('storage').setLevel(logging.INFO)
def batched(iterator, count=10):
batch = []
for value in iterator:
batch.append(value)
if len(batch) >= count:
yield batch
batch = []
if batch:
yield batch
def read_json_lines(stream):
for line in stream:
yield flask.json.loads(line)
def load_document(data):
doc = naturasites.schema.SpaDoc(data)
assert doc.validate(), '%s does not validate' % data['section1']['code']
assert doc.value == data, 'failed round-tripping the json data'
return doc
db = get_db(create_app())
for batch in batched(load_document(d) for d in read_json_lines(sys.stdin)):
db.save_document_batch(batch)
sys.stdout.write('.')
sys.stdout.flush()
print ' done'
def species_to_json():
values = set()
db = get_db()
search = naturasites.views._db_search(naturasites.schema.Search.from_flat({}), facets=True)
for d in search["docs"]:
doc = db.load_document(d["id"])
codes_and_labels = []
for specie in ("species_bird", "species_bird_extra", "species_mammal",
"species_reptile", "species_fish", "species_invertebrate",
"species_plant"):
for s in doc["section3"][specie]:
values.add((s["code"].value, s["name"].value.strip().lower()))
for s in doc["section3"]["species_other"]:
values.add((s["code"].value,
s["scientific_name"].value.strip().lower()))
print flask.json.dumps(dict(values), indent=4)
def species_to_json():
values = set()
db = get_db()
search = naturasites.views._db_search(naturasites.schema.Search.from_flat(
{}),
facets=True)
for d in search["docs"]:
doc = db.load_document(d["id"])
codes_and_labels = []
for specie in ("species_bird", "species_bird_extra", "species_mammal",
"species_reptile", "species_fish",
"species_invertebrate", "species_plant"):
for s in doc["section3"][specie]:
values.add((s["code"].value, s["name"].value.strip().lower()))
for s in doc["section3"]["species_other"]:
values.add(
(s["code"].value, s["scientific_name"].value.strip().lower()))
print flask.json.dumps(dict(values), indent=4)