Example #1
0
def load_items(register):
    d = {}
    dirname = os.path.join("data", REGISTRY_ENV, register)
    for file in os.listdir(dirname):
        if file.endswith(".yaml"):
            item = Item()
            item.yaml = open(os.path.join(dirname, file)).read()
            d[item[register]] = item
    return d
def test_set_of_tags_as_txt():
    item = Item(name='foo', fields={'z', 'b', 'c', 'z'})
    data = item.txt
    assert data == ('fields:\n'
                    '- b\n'
                    '- c\n'
                    '- z\n'
                    'name: foo\n')
def test_writer_one_item():
    out = io.StringIO()
    writer = Writer(out)
    item = Item(name="one")
    writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == '- name: one\n'
def test_writer_one_item():
    out = io.StringIO()
    writer = Writer(out, fieldnames=["name"])
    item = Item(name="one")
    writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == 'name\none\n'
def test_writer_one_list_item():
    out = io.StringIO()
    writer = Writer(out, fieldnames=["name"])
    item = Item(name=["one", "two", "three"])
    writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == 'name\none;two;three\n'
Example #6
0
def load_register(store,
                  name,
                  domain="register.gov.uk",
                  protocol="https",
                  path="download-register"):
    url = "{protocol}://{name}.{domain}/{path}".format(protocol=protocol,
                                                       name=name,
                                                       domain=domain,
                                                       path=path)

    response = cache.get(url)

    zipfile = ZipFile(BytesIO(response.content))

    for info in zipfile.infolist():
        if info.filename.startswith("item/"):
            item = Item()
            item.json = zipfile.open(info.filename).read().decode("utf-8")
            store.put(item)
Example #7
0
def test_simple_store():
    with TempDir() as tmp:
        store = FileStore(dir=tmp)
        text = "This is a test"
        item = Item(text=text)
        store.put(item)

        got = store.item(item.hash)
        assert item.hash == got.hash
        assert item.text == got.text
def test_ignore_private_as_tsv():
    item = Item()
    item._zero = "to be removed"
    item.one = "one value"
    item._two = "should be removed"
    item.two = "two value"
    item._three = "three"

    assert item.tsv == ("one\ttwo\n" "one value\ttwo value\n")
def test_ignore_private_as_csv():
    item = Item()
    item._zero = "to be removed"
    item.one = "one value"
    item._two = "should be removed"
    item.two = "two value"
    item._three = "three"

    data = item.csv
    assert data == ('"one","two"\r\n' '"one value","two value"\r\n')
def test_writer_many_items():
    out = io.StringIO()
    writer = Writer(out)
    for name in ['one', 'two', 'three']:
        item = Item(name=name)
        writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == ('{"name":"one"}\n'
                      '{"name":"two"}\n'
                      '{"name":"three"}\n')
Example #11
0
def test_writer_interface():
    item = Item()
    stream = io.StringIO()

    writer = Writer(stream)

    assert writer.stream == stream

    with pytest.raises(NotImplementedError):
        writer.write(item)

    writer.close()
Example #12
0
def test_ignore_private_as_json():
    item = Item()
    item._zero = "to be removed"
    item.one = "one value"
    item._two = "should be removed"
    item.two = "two value"
    item._three = "three"

    data = item.json
    assert data == ('{"one":"one value","two":"two value"}')
Example #13
0
def test_item_assignment():
    item = Item()
    item.foo = "Foo Value"
    item.bar = "Bar Value"
    item["kebab-value"] = "A Kebab"
    item["empty"] = ""

    assert item.foo == "Foo Value"
    assert item.bar == "Bar Value"
    assert item["bar"] == "Bar Value"
    assert item["kebab-value"] == "A Kebab"

    assert item.get("kebab-value") == "A Kebab"
    assert item.get("a-missing-value") is None
    assert item.get("empty") is None

    item.set("another", "Another Value")
    assert item.another == "Another Value"
Example #14
0
def test_item_assignment():
    item = Item()
    item.foo = "Foo Value"
    item.bar = "Bar Value"
    item['kebab-value'] = "A Kebab"
    item['empty'] = ''

    assert item.foo == "Foo Value"
    assert item.bar == "Bar Value"
    assert item['bar'] == "Bar Value"
    assert item['kebab-value'] == "A Kebab"

    assert item.get('kebab-value') == "A Kebab"
    assert item.get('a-missing-value') is None
    assert item.get('empty') is None

    item.set('another', "Another Value")
    assert item.another == "Another Value"
Example #15
0
def test_writer_many_items():
    out = io.StringIO()
    writer = Writer(out)
    for name in ["one", "two", "three"]:
        item = Item(name=name)
        writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == ("["
                      '{"name":"one"},'
                      '{"name":"two"},'
                      '{"name":"three"}]')
def test_writer_sparse_items():
    out = io.StringIO()
    writer = Writer(out, fieldnames=["name", "text", "address", "z"])
    for name in ['one', 'two', 'three']:
        item = Item(name=name, text="hello world")
        writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == ('name\ttext\taddress\tz\n'
                      'one\thello world\t\t\n'
                      'two\thello world\t\t\n'
                      'three\thello world\t\t\n')
Example #17
0
def test_writer_many_items():
    out = io.StringIO()
    writer = Writer(out, fieldnames=["name", "text"])
    for name in ["one", "two", "three"]:
        item = Item(name=name, text="hello world")
        writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == ('"name","text"\r\n'
                      '"one","hello world"\r\n'
                      '"two","hello world"\r\n'
                      '"three","hello world"\r\n')
def test_store():
    item = Item()
    empty_hash = item.hash
    store.put(item)

    item = Item(text='Foo Value')
    foo_hash = item.hash
    store.put(item)

    item = Item(text='Bar Value')
    bar_hash = item.hash
    store.put(item)

    item = store.item(empty_hash)
    assert item.hash == empty_hash

    item = store.item(foo_hash)
    assert item.hash == foo_hash
    assert item.text == 'Foo Value'

    item = store.item(bar_hash)
    assert item.hash == bar_hash
    assert item.text == 'Bar Value'
def test_postaladdress_as_tsv():
    item = Item()
    item.streetAddress = "Aviation House, 125 Kingsway"
    item.addressLocality = "Holborn"
    item.addressRegion = "London"
    item.postcode = "WC2B 6NH"
    item.addressCountry = "GB"

    data = item.tsv
    assert data == (
        "addressCountry\taddressLocality\taddressRegion\tpostcode\tstreetAddress\n"  # NOQA
        "GB\tHolborn\tLondon\tWC2B 6NH\tAviation House, 125 Kingsway\n"  # NOQA
    )
def test_postaladdress_as_csv():
    item = Item()
    item.streetAddress = "Aviation House, 125 Kingsway"
    item.addressLocality = "Holborn"
    item.addressRegion = "London"
    item.postcode = "WC2B 6NH"
    item.addressCountry = "GB"

    data = item.csv
    assert data == (
        '"addressCountry","addressLocality","addressRegion","postcode","streetAddress"\r\n'  # NOQA
        '"GB","Holborn","London","WC2B 6NH","Aviation House, 125 Kingsway"\r\n'  # NOQA
    )
def test_postaladdress_as_txt():
    item = Item()
    item.streetAddress = "Aviation House, 125 Kingsway"
    item.addressLocality = "Holborn"
    item.addressRegion = "London"
    item.postcode = "WC2B 6NH"
    item.addressCountry = "GB"

    data = item.txt
    assert data == ("addressCountry: GB\n"
                    "addressLocality: Holborn\n"
                    "addressRegion: London\n"
                    "postcode: WC2B 6NH\n"
                    "streetAddress: Aviation House, 125 Kingsway\n")
Example #22
0
def test_postaladdress_as_jsonl():
    item = Item()
    item.streetAddress = "Aviation House, 125 Kingsway"
    item.addressLocality = "Holborn"
    item.addressRegion = "London"
    item.postcode = "WC2B 6NH"
    item.addressCountry = "GB"

    data = item.json
    assert data == ('{"addressCountry":"GB",'
                    '"addressLocality":"Holborn",'
                    '"addressRegion":"London",'
                    '"postcode":"WC2B 6NH",'
                    '"streetAddress":"Aviation House, 125 Kingsway"}')
Example #23
0
def test_postaladdress_as_primitive():
    item = Item()
    item.streetAddress = "Aviation House, 125 Kingsway"
    item.addressLocality = "Holborn"
    item.addressRegion = "London"
    item.postcode = "WC2B 6NH"
    item.addressCountry = "GB"

    assert item.primitive == {
        "addressCountry": "GB",
        "addressLocality": "Holborn",
        "addressRegion": "London",
        "postcode": "WC2B 6NH",
        "streetAddress": "Aviation House, 125 Kingsway",
    }
Example #24
0
def test_postaladdress_as_primitive():
    item = Item()
    item.streetAddress = "Aviation House, 125 Kingsway"
    item.addressLocality = "Holborn"
    item.addressRegion = "London"
    item.postcode = "WC2B 6NH"
    item.addressCountry = "GB"

    assert item.primitive == {
        'addressCountry': 'GB',
        'addressLocality': 'Holborn',
        'addressRegion': 'London',
        'postcode': 'WC2B 6NH',
        'streetAddress': 'Aviation House, 125 Kingsway'
    }
Example #25
0
def test_writer_many_items():
    out = io.StringIO()
    writer = Writer(out)
    for name in ["one", "two", "three"]:
        item = Item(name=name, text="hello world")
        writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == ("- name: one\n"
                      "  text: hello world\n"
                      "\n"
                      "- name: two\n"
                      "  text: hello world\n"
                      "\n"
                      "- name: three\n"
                      "  text: hello world\n")
def test_writer_many_items():
    out = io.StringIO()
    writer = Writer(out)
    for name in ['one', 'two', 'three']:
        item = Item(name=name, text="hello world")
        writer.write(item)
    writer.close()

    string = out.getvalue()
    assert string == ('- name: one\n'
                      '  text: hello world\n'
                      '\n'
                      '- name: two\n'
                      '  text: hello world\n'
                      '\n'
                      '- name: three\n'
                      '  text: hello world\n')
#!/usr/bin/env python

# create a single jsonl file from individual register entries

import sys
import os
from openregister import Item
from openregister.representations.jsonl import Writer

env = sys.argv[1] or "alpha"
register = sys.argv[2] or "register"
dirname = os.path.join("data", env, register)

writer = Writer(sys.stdout)

for file in os.listdir(dirname):
    if file.endswith(".yaml"):
        item = Item()
        item.yaml = open(os.path.join(dirname, file)).read()
        writer.write(item)

writer.close()
Example #28
0

# Country,Competent Authority,Approval Number,Trading name,Address 1,Address 2,Adress 3,Town/Region,Postcode,Section 0,Section I,Section II,Section III,Section IV,Section V,Section VI,Section VII,Section VIII,Section IX,Section X,Section XI,Section XII,Section XIII,Section XIV,Section XV,Section XVI,Species,Remarks,Address withheld,uprn,lat,lng,match_score,match_description

fields = ["food-premises", "premises", "name", "business", "start-date", "end-date"]

writer = Writer(sys.stdout, fields)

reader = csv.DictReader(sys.stdin)
for num, row in enumerate(reader):
    uprn = row["uprn"]
    name = row["Trading name"]

    if uprn and row["Address withheld"] != "Y":

        item = Item()
        item.__dict__["food-premises"] = uprn

        if uprn in uprns:
            item.premises = uprns[uprn]

        if name in companies:
            if companies[name]:
                item.business = "company:" + companies[name]

        item.__dict__["end-date"] = ""
        item.name = row["Trading name"]

        writer.write(item)

writer.close()
Example #29
0
import pytest
import numbers
from openregister import Item
from openregister.store import Store

store = Store()
item = Item()


def test_store_interface():
    assert isinstance(store.page_size, numbers.Integral)

    with pytest.raises(NotImplementedError):
        store.put(item)

    with pytest.raises(NotImplementedError):
        store.add(item)

    with pytest.raises(NotImplementedError):
        store.item(item.hash)

    with pytest.raises(NotImplementedError):
        store.items()

    with pytest.raises(NotImplementedError):
        store.entry(1)

    with pytest.raises(NotImplementedError):
        store.entries()

    with pytest.raises(NotImplementedError):
def test_idempotent_put():
    item = Item(text='Idempotent?')
    store.put(item)
    store.put(item)
    store.put(item)
def test_tags():
    item = Item()
    item.tags = {'one', 'two', 'three'}
    item = store.put(item)
Example #32
0
    def open(self, prefix):
        self.prefix = prefix
        filename = os.path.join(self.dirname, self.prefix + '.tsv')
        self.writer = Writer(open(filename, "w"), self.fieldnames)
        print("writing %s .." % filename)

    def close(self):
        if self.writer:
            self.writer.close()


reader = csv.DictReader(sys.stdin)
reader.fieldnames = [field.strip() for field in reader.fieldnames]

writer = PostcodeWriter(dirname)

for row in reader:

    item = Item()
    item.postcode = row['pcds']

    try:
        item.point = osgb_point(row['oseast1m'], row['osnrth1m'])
    except TypeError:
        pass

    writer.write(item)

writer.close()
def test_counting():
    clear_db(mongo_uri, store.db.name)
    store.put(Item(text='one'))
Example #34
0
#!/usr/bin/env python

import os
import glob
from openregister import Item
from openregister.representations.tsv import reader as tsv_reader
from openregister.stores.mongodb import MongoStore

mongo_host = os.getenv('DB_PORT_27017_TCP_ADDR', '127.0.0.1')
mongo_uri = 'mongodb://%s:27017/landregistry' % mongo_host

# load register register
store = MongoStore(mongo_uri, prefix="register_")

for filename in glob.glob('data/register/*.yaml'):
    item = Item()
    item.yaml = open(filename)
    store.add(item)


# load company register
store = MongoStore(mongo_uri, prefix="company_")

for item in tsv_reader(open('data/company/companies.tsv')):
    store.add(item)

store = MongoStore(mongo_uri, prefix="land_")
for item in tsv_reader(open('demodata/land/lands.tsv')):
    store.add(item)

store = MongoStore(mongo_uri, prefix="land_title_")
Example #35
0
def test_tags():
    with TempDir() as tmp:
        store = FileStore(dir=tmp)
        item = Item()
        item.tags = {"one", "two", "three"}
        item = store.put(item)