Example #1
0
def import_mjson():
    logging.getLogger('storage').setLevel(logging.INFO)

    def batched(iterator, count=10):
        batch = []
        for value in iterator:
            batch.append(value)
            if len(batch) >= count:
                yield batch
                batch = []
        if batch:
            yield batch

    def read_json_lines(stream):
        for line in stream:
            yield flask.json.loads(line)

    def load_document(data):
        doc = naturasites.schema.SpaDoc(data)
        assert doc.validate(
        ), '%s does not validate' % data['section1']['code']
        assert doc.value == data, 'failed round-tripping the json data'
        return doc

    db = get_db(create_app())

    for batch in batched(load_document(d) for d in read_json_lines(sys.stdin)):
        db.save_document_batch(batch)
        sys.stdout.write('.')
        sys.stdout.flush()

    print ' done'
Example #2
0
def import_mjson():
    logging.getLogger('storage').setLevel(logging.INFO)

    def batched(iterator, count=10):
        batch = []
        for value in iterator:
            batch.append(value)
            if len(batch) >= count:
                yield batch
                batch = []
        if batch:
            yield batch

    def read_json_lines(stream):
        for line in stream:
            yield flask.json.loads(line)

    def load_document(data):
        doc = naturasites.schema.SpaDoc(data)
        assert doc.validate(), '%s does not validate' % data['section1']['code']
        assert doc.value == data, 'failed round-tripping the json data'
        return doc

    db = get_db(create_app())

    for batch in batched(load_document(d) for d in read_json_lines(sys.stdin)):
        db.save_document_batch(batch)
        sys.stdout.write('.')
        sys.stdout.flush()

    print ' done'
Example #3
0
def species_to_json():
    values = set()
    db = get_db()

    search =  naturasites.views._db_search(naturasites.schema.Search.from_flat({}), facets=True)
    for d in search["docs"]:
        doc = db.load_document(d["id"])
        codes_and_labels = []

        for specie in ("species_bird", "species_bird_extra", "species_mammal",
                       "species_reptile", "species_fish", "species_invertebrate",
                       "species_plant"):
            for s in doc["section3"][specie]:
                values.add((s["code"].value, s["name"].value.strip().lower()))

        for s in doc["section3"]["species_other"]:
            values.add((s["code"].value,
                        s["scientific_name"].value.strip().lower()))

    print flask.json.dumps(dict(values), indent=4)
Example #4
0
def species_to_json():
    values = set()
    db = get_db()

    search = naturasites.views._db_search(naturasites.schema.Search.from_flat(
        {}),
                                          facets=True)
    for d in search["docs"]:
        doc = db.load_document(d["id"])
        codes_and_labels = []

        for specie in ("species_bird", "species_bird_extra", "species_mammal",
                       "species_reptile", "species_fish",
                       "species_invertebrate", "species_plant"):
            for s in doc["section3"][specie]:
                values.add((s["code"].value, s["name"].value.strip().lower()))

        for s in doc["section3"]["species_other"]:
            values.add(
                (s["code"].value, s["scientific_name"].value.strip().lower()))

    print flask.json.dumps(dict(values), indent=4)