def load_items(register):
d = {}
dirname = os.path.join("data", REGISTRY_ENV, register)
for file in os.listdir(dirname):
if file.endswith(".yaml"):
item = Item()
item.yaml = open(os.path.join(dirname, file)).read()
d[item[register]] = item
return d
def test_set_of_tags_as_txt():
item = Item(name='foo', fields={'z', 'b', 'c', 'z'})
data = item.txt
assert data == ('fields:\n'
'- b\n'
'- c\n'
'- z\n'
'name: foo\n')
def test_writer_one_item():
out = io.StringIO()
writer = Writer(out)
item = Item(name="one")
writer.write(item)
writer.close()
string = out.getvalue()
assert string == '- name: one\n'
def test_writer_one_item():
out = io.StringIO()
writer = Writer(out, fieldnames=["name"])
item = Item(name="one")
writer.write(item)
writer.close()
string = out.getvalue()
assert string == 'name\none\n'
def test_writer_one_list_item():
out = io.StringIO()
writer = Writer(out, fieldnames=["name"])
item = Item(name=["one", "two", "three"])
writer.write(item)
writer.close()
string = out.getvalue()
assert string == 'name\none;two;three\n'
def load_register(store,
name,
domain="register.gov.uk",
protocol="https",
path="download-register"):
url = "{protocol}://{name}.{domain}/{path}".format(protocol=protocol,
name=name,
domain=domain,
path=path)
response = cache.get(url)
zipfile = ZipFile(BytesIO(response.content))
for info in zipfile.infolist():
if info.filename.startswith("item/"):
item = Item()
item.json = zipfile.open(info.filename).read().decode("utf-8")
store.put(item)
def test_simple_store():
with TempDir() as tmp:
store = FileStore(dir=tmp)
text = "This is a test"
item = Item(text=text)
store.put(item)
got = store.item(item.hash)
assert item.hash == got.hash
assert item.text == got.text
def test_ignore_private_as_tsv():
item = Item()
item._zero = "to be removed"
item.one = "one value"
item._two = "should be removed"
item.two = "two value"
item._three = "three"
assert item.tsv == ("one\ttwo\n" "one value\ttwo value\n")
def test_ignore_private_as_csv():
item = Item()
item._zero = "to be removed"
item.one = "one value"
item._two = "should be removed"
item.two = "two value"
item._three = "three"
data = item.csv
assert data == ('"one","two"\r\n' '"one value","two value"\r\n')
def test_writer_many_items():
out = io.StringIO()
writer = Writer(out)
for name in ['one', 'two', 'three']:
item = Item(name=name)
writer.write(item)
writer.close()
string = out.getvalue()
assert string == ('{"name":"one"}\n'
'{"name":"two"}\n'
'{"name":"three"}\n')
def test_writer_interface():
item = Item()
stream = io.StringIO()
writer = Writer(stream)
assert writer.stream == stream
with pytest.raises(NotImplementedError):
writer.write(item)
writer.close()
def test_ignore_private_as_json():
item = Item()
item._zero = "to be removed"
item.one = "one value"
item._two = "should be removed"
item.two = "two value"
item._three = "three"
data = item.json
assert data == ('{"one":"one value","two":"two value"}')
def test_item_assignment():
item = Item()
item.foo = "Foo Value"
item.bar = "Bar Value"
item["kebab-value"] = "A Kebab"
item["empty"] = ""
assert item.foo == "Foo Value"
assert item.bar == "Bar Value"
assert item["bar"] == "Bar Value"
assert item["kebab-value"] == "A Kebab"
assert item.get("kebab-value") == "A Kebab"
assert item.get("a-missing-value") is None
assert item.get("empty") is None
item.set("another", "Another Value")
assert item.another == "Another Value"
def test_item_assignment():
item = Item()
item.foo = "Foo Value"
item.bar = "Bar Value"
item['kebab-value'] = "A Kebab"
item['empty'] = ''
assert item.foo == "Foo Value"
assert item.bar == "Bar Value"
assert item['bar'] == "Bar Value"
assert item['kebab-value'] == "A Kebab"
assert item.get('kebab-value') == "A Kebab"
assert item.get('a-missing-value') is None
assert item.get('empty') is None
item.set('another', "Another Value")
assert item.another == "Another Value"
def test_writer_many_items():
out = io.StringIO()
writer = Writer(out)
for name in ["one", "two", "three"]:
item = Item(name=name)
writer.write(item)
writer.close()
string = out.getvalue()
assert string == ("["
'{"name":"one"},'
'{"name":"two"},'
'{"name":"three"}]')
def test_writer_sparse_items():
out = io.StringIO()
writer = Writer(out, fieldnames=["name", "text", "address", "z"])
for name in ['one', 'two', 'three']:
item = Item(name=name, text="hello world")
writer.write(item)
writer.close()
string = out.getvalue()
assert string == ('name\ttext\taddress\tz\n'
'one\thello world\t\t\n'
'two\thello world\t\t\n'
'three\thello world\t\t\n')
def test_writer_many_items():
out = io.StringIO()
writer = Writer(out, fieldnames=["name", "text"])
for name in ["one", "two", "three"]:
item = Item(name=name, text="hello world")
writer.write(item)
writer.close()
string = out.getvalue()
assert string == ('"name","text"\r\n'
'"one","hello world"\r\n'
'"two","hello world"\r\n'
'"three","hello world"\r\n')
def test_store():
item = Item()
empty_hash = item.hash
store.put(item)
item = Item(text='Foo Value')
foo_hash = item.hash
store.put(item)
item = Item(text='Bar Value')
bar_hash = item.hash
store.put(item)
item = store.item(empty_hash)
assert item.hash == empty_hash
item = store.item(foo_hash)
assert item.hash == foo_hash
assert item.text == 'Foo Value'
item = store.item(bar_hash)
assert item.hash == bar_hash
assert item.text == 'Bar Value'
def test_postaladdress_as_tsv():
item = Item()
item.streetAddress = "Aviation House, 125 Kingsway"
item.addressLocality = "Holborn"
item.addressRegion = "London"
item.postcode = "WC2B 6NH"
item.addressCountry = "GB"
data = item.tsv
assert data == (
"addressCountry\taddressLocality\taddressRegion\tpostcode\tstreetAddress\n" # NOQA
"GB\tHolborn\tLondon\tWC2B 6NH\tAviation House, 125 Kingsway\n" # NOQA
)
def test_postaladdress_as_csv():
item = Item()
item.streetAddress = "Aviation House, 125 Kingsway"
item.addressLocality = "Holborn"
item.addressRegion = "London"
item.postcode = "WC2B 6NH"
item.addressCountry = "GB"
data = item.csv
assert data == (
'"addressCountry","addressLocality","addressRegion","postcode","streetAddress"\r\n' # NOQA
'"GB","Holborn","London","WC2B 6NH","Aviation House, 125 Kingsway"\r\n' # NOQA
)
def test_postaladdress_as_txt():
item = Item()
item.streetAddress = "Aviation House, 125 Kingsway"
item.addressLocality = "Holborn"
item.addressRegion = "London"
item.postcode = "WC2B 6NH"
item.addressCountry = "GB"
data = item.txt
assert data == ("addressCountry: GB\n"
"addressLocality: Holborn\n"
"addressRegion: London\n"
"postcode: WC2B 6NH\n"
"streetAddress: Aviation House, 125 Kingsway\n")
def test_postaladdress_as_jsonl():
item = Item()
item.streetAddress = "Aviation House, 125 Kingsway"
item.addressLocality = "Holborn"
item.addressRegion = "London"
item.postcode = "WC2B 6NH"
item.addressCountry = "GB"
data = item.json
assert data == ('{"addressCountry":"GB",'
'"addressLocality":"Holborn",'
'"addressRegion":"London",'
'"postcode":"WC2B 6NH",'
'"streetAddress":"Aviation House, 125 Kingsway"}')
def test_postaladdress_as_primitive():
item = Item()
item.streetAddress = "Aviation House, 125 Kingsway"
item.addressLocality = "Holborn"
item.addressRegion = "London"
item.postcode = "WC2B 6NH"
item.addressCountry = "GB"
assert item.primitive == {
"addressCountry": "GB",
"addressLocality": "Holborn",
"addressRegion": "London",
"postcode": "WC2B 6NH",
"streetAddress": "Aviation House, 125 Kingsway",
}
def test_postaladdress_as_primitive():
item = Item()
item.streetAddress = "Aviation House, 125 Kingsway"
item.addressLocality = "Holborn"
item.addressRegion = "London"
item.postcode = "WC2B 6NH"
item.addressCountry = "GB"
assert item.primitive == {
'addressCountry': 'GB',
'addressLocality': 'Holborn',
'addressRegion': 'London',
'postcode': 'WC2B 6NH',
'streetAddress': 'Aviation House, 125 Kingsway'
}
def test_writer_many_items():
out = io.StringIO()
writer = Writer(out)
for name in ["one", "two", "three"]:
item = Item(name=name, text="hello world")
writer.write(item)
writer.close()
string = out.getvalue()
assert string == ("- name: one\n"
" text: hello world\n"
"\n"
"- name: two\n"
" text: hello world\n"
"\n"
"- name: three\n"
" text: hello world\n")
def test_writer_many_items():
out = io.StringIO()
writer = Writer(out)
for name in ['one', 'two', 'three']:
item = Item(name=name, text="hello world")
writer.write(item)
writer.close()
string = out.getvalue()
assert string == ('- name: one\n'
' text: hello world\n'
'\n'
'- name: two\n'
' text: hello world\n'
'\n'
'- name: three\n'
' text: hello world\n')
#!/usr/bin/env python
# create a single jsonl file from individual register entries
import sys
import os
from openregister import Item
from openregister.representations.jsonl import Writer
env = sys.argv[1] or "alpha"
register = sys.argv[2] or "register"
dirname = os.path.join("data", env, register)
writer = Writer(sys.stdout)
for file in os.listdir(dirname):
if file.endswith(".yaml"):
item = Item()
item.yaml = open(os.path.join(dirname, file)).read()
writer.write(item)
writer.close()
# Country,Competent Authority,Approval Number,Trading name,Address 1,Address 2,Adress 3,Town/Region,Postcode,Section 0,Section I,Section II,Section III,Section IV,Section V,Section VI,Section VII,Section VIII,Section IX,Section X,Section XI,Section XII,Section XIII,Section XIV,Section XV,Section XVI,Species,Remarks,Address withheld,uprn,lat,lng,match_score,match_description
fields = ["food-premises", "premises", "name", "business", "start-date", "end-date"]
writer = Writer(sys.stdout, fields)
reader = csv.DictReader(sys.stdin)
for num, row in enumerate(reader):
uprn = row["uprn"]
name = row["Trading name"]
if uprn and row["Address withheld"] != "Y":
item = Item()
item.__dict__["food-premises"] = uprn
if uprn in uprns:
item.premises = uprns[uprn]
if name in companies:
if companies[name]:
item.business = "company:" + companies[name]
item.__dict__["end-date"] = ""
item.name = row["Trading name"]
writer.write(item)
writer.close()
import pytest
import numbers
from openregister import Item
from openregister.store import Store
store = Store()
item = Item()
def test_store_interface():
assert isinstance(store.page_size, numbers.Integral)
with pytest.raises(NotImplementedError):
store.put(item)
with pytest.raises(NotImplementedError):
store.add(item)
with pytest.raises(NotImplementedError):
store.item(item.hash)
with pytest.raises(NotImplementedError):
store.items()
with pytest.raises(NotImplementedError):
store.entry(1)
with pytest.raises(NotImplementedError):
store.entries()
with pytest.raises(NotImplementedError):
def test_idempotent_put():
item = Item(text='Idempotent?')
store.put(item)
store.put(item)
store.put(item)
def test_tags():
item = Item()
item.tags = {'one', 'two', 'three'}
item = store.put(item)
def open(self, prefix):
self.prefix = prefix
filename = os.path.join(self.dirname, self.prefix + '.tsv')
self.writer = Writer(open(filename, "w"), self.fieldnames)
print("writing %s .." % filename)
def close(self):
if self.writer:
self.writer.close()
reader = csv.DictReader(sys.stdin)
reader.fieldnames = [field.strip() for field in reader.fieldnames]
writer = PostcodeWriter(dirname)
for row in reader:
item = Item()
item.postcode = row['pcds']
try:
item.point = osgb_point(row['oseast1m'], row['osnrth1m'])
except TypeError:
pass
writer.write(item)
writer.close()
def test_counting():
clear_db(mongo_uri, store.db.name)
store.put(Item(text='one'))
#!/usr/bin/env python
import os
import glob
from openregister import Item
from openregister.representations.tsv import reader as tsv_reader
from openregister.stores.mongodb import MongoStore
mongo_host = os.getenv('DB_PORT_27017_TCP_ADDR', '127.0.0.1')
mongo_uri = 'mongodb://%s:27017/landregistry' % mongo_host
# load register register
store = MongoStore(mongo_uri, prefix="register_")
for filename in glob.glob('data/register/*.yaml'):
item = Item()
item.yaml = open(filename)
store.add(item)
# load company register
store = MongoStore(mongo_uri, prefix="company_")
for item in tsv_reader(open('data/company/companies.tsv')):
store.add(item)
store = MongoStore(mongo_uri, prefix="land_")
for item in tsv_reader(open('demodata/land/lands.tsv')):
store.add(item)
store = MongoStore(mongo_uri, prefix="land_title_")
def test_tags():
with TempDir() as tmp:
store = FileStore(dir=tmp)
item = Item()
item.tags = {"one", "two", "three"}
item = store.put(item)