def test_inherit_existing(self):
"""Inherit from existing entry
/root (height=50)
/group (height=100)
/subgroup <-- Inherits above
"""
group = os.path.join(self.root_path, 'group')
subgroup = os.path.join(group, 'subgroup')
os.makedirs(subgroup)
base_location = self.root
group_location = om.Location(group)
subgroup_location = om.Location(subgroup)
base_value = om.Entry('height', value=50, parent=base_location)
group_value = om.Entry('height', value=100, parent=group_location)
om.flush(base_value)
om.flush(group_value)
# Get metadata from location where metadata doesn't exist
height = om.Entry('height', parent=subgroup_location)
om.inherit(height)
self.assertEquals(height.value, 100)
def test_convert(self):
"""contert() is a shorthand for creating location and entry sep."""
entry = om.Entry('custom.int', value=10, parent=self.root)
om.flush(entry)
entry = om.convert(entry.path.as_str)
self.assertEquals(entry.value, 10)
def test_convert(self):
"""contert() is a shorthand for creating location and entry sep."""
entry = om.Entry('custom.int', value=10, parent=self.root)
om.flush(entry)
entry = om.convert(entry.path.as_str)
self.assertEquals(entry.value, 10)
def test_removal(self):
removed = om.Entry('removed', value=1, parent=self.root)
om.flush(removed)
self.assertTrue(os.path.isfile(removed.path.as_str))
om.recycle(removed)
self.assertFalse(os.path.exists(removed.path.as_str))
def why_simple_data():
"""Store data that we might need to change later"""
location = om.Location(path)
girlfriend = om.Dataset('girlfriend', parent=location)
girlfriend.value = 'Sweet Heartsson'
om.flush(girlfriend)
om.clear(path)
def test_add_entries_to_nongroup(self):
"""Add entries to nongroup
Adding entries to an entry that isn't a group will
cast it to a group.
"""
nongroup_entry = om.Entry('nongroup',
value='A string',
parent=self.root)
self.assertTrue(nongroup_entry.type == 'string')
om.flush(nongroup_entry)
invalid_child = om.Entry('invalid_child',
value='a string',
parent=nongroup_entry)
# By adding a child, the nongroup becomes a group.
# Just like it would in a dynamic programming language:
# >>> myint = 5
# >>> myint = list()
self.assertEquals(nongroup_entry.type, 'dict')
om.flush(nongroup_entry)
self.assertTrue(os.path.exists(nongroup_entry.path.as_str))
om.pull(nongroup_entry)
self.assertTrue(os.path.exists(invalid_child.path.as_str))
self.assertTrue(os.path.isdir(nongroup_entry.path.as_str))
self.assertEquals(nongroup_entry.type, 'dict')
def storing_collection_as_blobs():
"""
Open Metadata can store any arbitrary data as blobs. Blobs are stored
together with its parent; i.e. in the simples of situation, blobs are
copied from their source into the Open Metadata hierarchy.
__________________ ____________________________
| | | |
| /root/source.bin |---->| /location/.meta/source.bin |
|__________________| |____________________________|
"""
raise NotImplementedError
path = '/project/asset'
location = om.Location(path)
myvalue = om.Entry('myvalue', parent=location)
myvalue.value = '/path/to/myvalue.json'
myvalue.isblob = True
om.flush(myvalue)
# Retreiving stored blob
myvalue = om.read(path, 'myvalue')
myvalue.path
# --> '/project/asset/.meta/myvalue.json'
assert myvalue.value == myvalue.path
def test_change_type_by_overwriting(self):
"""Types in Open Metadata are represented on disk as suffixes.
However a suffix is an implementation detail and should not
be modified by hand.
The equivalence in dynamic programming languages is this:
>> myint = 5
>> myint = "I'm a string now"
On disk, myint would initially be stored as `myint.int` but
after having been reassigned a string value, it would be stored
as `myint.string`. The point to take home being that suffixes
are a serialisation of type and dictates how the file is to be
read.
"""
height = om.Entry('height', value=10, parent=self.root)
om.flush(height)
self.assertEquals(height.type, 'int')
height = om.Entry('height', value=11.1, parent=self.root)
om.flush(height)
self.assertEquals(height.type, 'float')
def test_convert_withmissingsuffix(self):
entry = om.Entry('custom_missing.int', value=10, parent=self.root)
om.flush(entry)
new_path = entry.path.copy(suffix='string')
entry = om.convert(new_path.as_str)
self.assertEquals(entry.value, 10)
def test_change_type_by_overwriting(self):
"""Types in Open Metadata are represented on disk as suffixes.
However a suffix is an implementation detail and should not
be modified by hand.
The equivalence in dynamic programming languages is this:
>> myint = 5
>> myint = "I'm a string now"
On disk, myint would initially be stored as `myint.int` but
after having been reassigned a string value, it would be stored
as `myint.string`. The point to take home being that suffixes
are a serialisation of type and dictates how the file is to be
read.
"""
height = om.Entry('height', value=10, parent=self.root)
om.flush(height)
self.assertEquals(height.type, 'int')
height = om.Entry('height', value=11.1, parent=self.root)
om.flush(height)
self.assertEquals(height.type, 'float')
def test_removal(self):
removed = om.Entry('removed', value=1, parent=self.root)
om.flush(removed)
self.assertTrue(os.path.isfile(removed.path.as_str))
om.recycle(removed)
self.assertFalse(os.path.exists(removed.path.as_str))
def test_add_entries_to_nongroup(self):
"""Add entries to nongroup
Adding entries to an entry that isn't a group will
cast it to a group.
"""
nongroup_entry = om.Entry('nongroup',
value='A string',
parent=self.root)
self.assertTrue(nongroup_entry.type == 'string')
om.flush(nongroup_entry)
invalid_child = om.Entry('invalid_child',
value='a string',
parent=nongroup_entry)
# By adding a child, the nongroup becomes a group.
# Just like it would in a dynamic programming language:
# >>> myint = 5
# >>> myint = list()
self.assertEquals(nongroup_entry.type, 'dict')
om.flush(nongroup_entry)
self.assertTrue(os.path.exists(nongroup_entry.path.as_str))
om.pull(nongroup_entry)
self.assertTrue(os.path.exists(invalid_child.path.as_str))
self.assertTrue(os.path.isdir(nongroup_entry.path.as_str))
self.assertEquals(nongroup_entry.type, 'dict')
def test_convert_withmissingsuffix(self):
entry = om.Entry('custom_missing.int', value=10, parent=self.root)
om.flush(entry)
new_path = entry.path.copy(suffix='string')
entry = om.convert(new_path.as_str)
self.assertEquals(entry.value, 10)
def test_new_group(self):
"""Write a new group"""
entry = om.Entry('a group.dict', parent=self.root)
om.flush(entry)
om.pull(entry)
self.assertEquals(entry.type, 'dict')
self.assertTrue(os.path.isdir(entry.path.as_str))
def test_new_group(self):
"""Write a new group"""
entry = om.Entry('a group.dict', parent=self.root)
om.flush(entry)
om.pull(entry)
self.assertEquals(entry.type, 'dict')
self.assertTrue(os.path.isdir(entry.path.as_str))
def test_dict(self):
"""Write dict"""
name = 'mydict.dict'
dic = om.Entry(name, parent=self.root)
om.flush(dic)
om.pull(dic)
self.assertEqual(dic.type, 'dict')
self.assertEqual(dic.name, 'mydict')
def test_absolutename(self):
"""If entry exists, suffix will be implied by find()"""
entry = om.Entry('custom.int', value=10, parent=self.root)
om.flush(entry)
entry = om.Entry('custom.string', value="Hello", parent=self.root)
om.pull(entry)
self.assertEquals(entry.type, 'int')
def test_absolutename(self):
"""If entry exists, suffix will be implied by find()"""
entry = om.Entry('custom.int', value=10, parent=self.root)
om.flush(entry)
entry = om.Entry('custom.string', value="Hello", parent=self.root)
om.pull(entry)
self.assertEquals(entry.type, 'int')
def test_int(self):
name = 'integer'
value = 10
integer = om.Entry(name, value=value, parent=self.root)
om.flush(integer)
om.pull(integer)
self.assertEquals(integer.value, value)
self.assertEquals(integer.name, name)
self.assertEquals(integer.type, 'int')
def test_removal_group(self):
removed = om.Entry('removed.dict', parent=self.root)
om.Entry('child', value=1, parent=removed)
self.assertEquals(removed.type, 'dict')
om.flush(removed)
self.assertTrue(os.path.isdir(removed.path.as_str))
om.recycle(removed)
self.assertFalse(os.path.exists(removed.path.as_str))
def test_removal_group(self):
removed = om.Entry('removed.dict', parent=self.root)
om.Entry('child', value=1, parent=removed)
self.assertEquals(removed.type, 'dict')
om.flush(removed)
self.assertTrue(os.path.isdir(removed.path.as_str))
om.recycle(removed)
self.assertFalse(os.path.exists(removed.path.as_str))
def test_new_group_with_content(self):
"""The alternative is to add other entries to is"""
entry = om.Entry('a group', parent=self.root)
for key, value in self.data.iteritems():
om.Entry(key, value=value, parent=entry)
om.flush(entry)
self.assertTrue(entry.type == 'dict')
self.assertTrue(os.path.exists(entry.path.as_str))
def test_new_group_with_content(self):
"""The alternative is to add other entries to is"""
entry = om.Entry('a group', parent=self.root)
for key, value in self.data.iteritems():
om.Entry(key, value=value, parent=entry)
om.flush(entry)
self.assertTrue(entry.type == 'dict')
self.assertTrue(os.path.exists(entry.path.as_str))
def test_flush_existing(self):
"""Overwrite existing entry"""
# Make it exist
standard_int = om.Entry('standard_int', value=10, parent=self.root)
om.flush(standard_int)
self.assertTrue(os.path.exists(standard_int.path.as_str))
# Then flush it again
standard_int = om.Entry('standard_int', value=15, parent=self.root)
om.flush(standard_int)
om.pull(standard_int)
self.assertEquals(standard_int.value, 15)
def test_flush_existing(self):
"""Overwrite existing entry"""
# Make it exist
standard_int = om.Entry('standard_int', value=10, parent=self.root)
om.flush(standard_int)
self.assertTrue(os.path.exists(standard_int.path.as_str))
# Then flush it again
standard_int = om.Entry('standard_int', value=15, parent=self.root)
om.flush(standard_int)
om.pull(standard_int)
self.assertEquals(standard_int.value, 15)
def flush_multiple(self):
parent = om.Entry('parent', parent=self.root)
for key, value in self.data.iteritems():
om.Entry(key, value=value, parent=parent)
om.flush(parent)
# Read data back from disk and re-build it
om.pull(parent)
pulled_data = dict()
for child in parent:
om.pull(child)
pulled_data[child.path.basename] = child.value
self.assertEquals(self.data, pulled_data)
def flush_multiple(self):
parent = om.Entry('parent', parent=self.root)
for key, value in self.data.iteritems():
om.Entry(key, value=value, parent=parent)
om.flush(parent)
# Read data back from disk and re-build it
om.pull(parent)
pulled_data = dict()
for child in parent:
om.pull(child)
pulled_data[child.path.basename] = child.value
self.assertEquals(self.data, pulled_data)
def storing_collection_as_open_metadata():
"""
While blobs facilitate storage of any arbitrary data, Open Metadata also
has the notion of Open Metadata-types.
Open Metadata-types offer extended features for plain-data, such as text,
numbers, dates and matrices in that they may be added, subtracted, version-
controlled and eventually end up in user interfaces as views and editors
relevant to their specific type.
_______________________ ___________________________________________
| | | |
| ['entry1', 'entry2' ] |---->| /location/.meta/myvalue.list/0.string |
|_______________________| | |___________________________________________|
|
| ___________________________________________
| | |
\-->| /location/.meta/myvalue.list/1.string |
|___________________________________________|
The advantage of each ultimately depends on your use-case, but in short:
- If control > performance, use Open Metadata-types
- If control < performance, use Blobs
"""
raise NotImplementedError
location = om.Location(path)
myvalue = om.Entry('myvalue', parent=location)
myvalue.value = ['entry1', 'entry2']
om.flush(myvalue)
# Retreiving stored blob
myvalue = om.read(path, 'myvalue')
myvalue == ['entry1', 'entry2']
assert myvalue.value != myvalue.path
def test_integration(self):
"""Test a combination of features"""
entry = om.Entry('test.string', value="Hello", parent=self.root)
child = om.Entry('child.int', value=1, parent=entry)
self.assertEquals(entry.type, 'dict')
om.flush(entry)
self.assertTrue(os.path.exists(entry.path.as_str))
om.pull(entry)
self.assertEquals(entry.type, 'dict')
entry.value = "Hello"
self.assertEquals(entry.type, 'string')
self.assertEquals(entry.value, "Hello")
om.flush(entry)
om.pull(entry)
self.assertFalse(os.path.exists(child.path.as_str))
child = om.Entry('child.int', value=1, parent=entry)
om.flush(entry)
self.assertEquals(om.read(self.root_path, 'test/child'), 1)
om.write(self.root_path, '/test/child', 2)
self.assertEquals(om.read(self.root_path, 'test/child'), 2)
om.write(self.root_path, '/root/test/another', 10)
self.assertEquals(om.read(self.root_path, 'root/test/another'), 10)
def test_integration(self):
"""Test a combination of features"""
entry = om.Entry('test.string', value="Hello", parent=self.root)
child = om.Entry('child.int', value=1, parent=entry)
self.assertEquals(entry.type, 'dict')
om.flush(entry)
self.assertTrue(os.path.exists(entry.path.as_str))
om.pull(entry)
self.assertEquals(entry.type, 'dict')
entry.value = "Hello"
self.assertEquals(entry.type, 'string')
self.assertEquals(entry.value, "Hello")
om.flush(entry)
om.pull(entry)
self.assertFalse(os.path.exists(child.path.as_str))
child = om.Entry('child.int', value=1, parent=entry)
om.flush(entry)
self.assertEquals(om.read(self.root_path, 'test/child'), 1)
om.write(self.root_path, '/test/child', 2)
self.assertEquals(om.read(self.root_path, 'test/child'), 2)
om.write(self.root_path, '/root/test/another', 10)
self.assertEquals(om.read(self.root_path, 'root/test/another'), 10)
def test_suffix_and_type_mismatch(self):
height = om.Entry('height.int', value=10.1, parent=self.root)
self.assertEquals(height.type, 'float')
om.flush(height)
om.pull(height)
self.assertEquals(height.type, 'float')
l1 = om.Entry('item1.string', value='a string value', parent=olist)
l2 = om.Entry('item2.bool', value=True, parent=olist)
l3 = om.Entry('item3.int', value=5, parent=olist)
# ..and a dictionary..
odict = om.Entry('mydict.dict', parent=olist)
# ..with two keys
key1 = om.Entry('key1.string', value='value', parent=odict)
# One of which, we neglect to specify a value-type.
# The value-type will be determined via the Python value-type <str>
key2 = om.Entry('key2', value='value', parent=odict)
# Finally, write it to disk.
om.flush(location)
# ----------- Read it back in
assert om.read(path) == ['mylist', 'simple_data']
assert om.read(path, 'mylist') == ['item2', 'item3', 'item1', 'mydict']
assert om.read(path, 'mylist/item2') is True
# ------------ Remove additions
om.clear(path)
# Add an entry with default value of type `bool`
l4 = om.Entry('item4.bool', parent=olist)
# ..and a dictionary..
odict = om.Entry('mydict.dict', parent=olist)
# ..with two keys
key1 = om.Entry('key1.string', value='value', parent=odict)
# One of which, we neglect to specify a value-type.
# The value-type will be determined via the Python value-type <str>
key2 = om.Entry('key2', value='value', parent=odict)
# Finally, write it to disk.
om.flush(location)
# ----------- Read it back in
assert om.read(root) == ['mylist', 'simple_data']
assert om.read(root,
'mylist') == ['item2', 'item3', 'item1', 'mydict', 'item4']
assert om.read(root, 'mylist/item2') is True
# ------------ Remove additions
finally:
shutil.rmtree(root)
def test_suffix_and_type_mismatch(self):
height = om.Entry('height.int', value=10.1, parent=self.root)
self.assertEquals(height.type, 'float')
om.flush(height)
om.pull(height)
self.assertEquals(height.type, 'float')
raise NotImplementedError
import os
import openmetadata as om
# Starting-point
path = os.path.expanduser(r'~/om')
location = om.Location(path)
# Add a regular string
password = om.Dataset('password.string', parent=location)
password.data = 'abcdef'
# Write first occurence of dataset
om.flush(password)
# Save version and update dataset
om.save(password)
password.data = 'Sandra Bullock'
om.flush(password)
assert om.read(path, 'password') == 'Sandra Bullock'
# Restore previous value from history
# *note: om.history() returns a Python generator.
imprint = om.history(password).next()
om.restore(imprint)
assert om.read(path, 'password') == 'Lucy Lui'
