def test_load_by_hash(self):
"""
Tests loading two different versions of the same
package using command.load and specifying the package
hash.
"""
# Old Version
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
_, contents = PackageStore.find_package(None, 'foo', 'package')
pkghash = hash_contents(contents)
# New Version
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/package', build_path)
command.ls()
load_pkg_new = command.load('foo/package')
load_pkg_old = command.load('foo/package', hash=pkghash)
assert load_pkg_new.foo
with self.assertRaises(AttributeError):
load_pkg_new.dataframes
assert load_pkg_old.dataframes
assert load_pkg_old.README
# Known failure cases
# At present load does not support extended package syntax
with self.assertRaises(command.CommandException):
command.load('foo/package:h:%s' % pkghash)
with self.assertRaises(command.CommandException):
command.load('foo/package:t:latest')
with self.assertRaises(command.CommandException):
command.load('foo/package:v:1.0.0')
def test_save(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package1', build_path)
from quilt.data.foo import package1
# Build an identical package
command.build('foo/package2', package1)
from quilt.data.foo import package2
teststore = PackageStore(self._store_dir)
contents1 = open(
os.path.join(teststore.package_path(None, 'foo', 'package1'),
Package.CONTENTS_DIR,
package1._package.get_hash())).read()
contents2 = open(
os.path.join(teststore.package_path(None, 'foo', 'package2'),
Package.CONTENTS_DIR,
package2._package.get_hash())).read()
assert contents1 == contents2
# Rename an attribute
package1.dataframes2 = package1.dataframes
del package1.dataframes
# Modify an existing dataframe
csv = package1.dataframes2.csv._data()
csv.at[0, 'Int0'] = 42
# Add a new dataframe
df = pd.DataFrame(dict(a=[1, 2, 3]))
package1._set(['new', 'df'], df)
assert package1.new.df._data() is df
# Add a new file
file_path = os.path.join(mydir, 'data/foo.csv')
package1._set(['new', 'file'], 'data/foo.csv', build_dir=mydir)
assert package1.new.file._data() == file_path
# Add a new group
package1._add_group('newgroup')
assert isinstance(package1.newgroup, GroupNode)
package1.newgroup._add_group('foo')
assert isinstance(package1.newgroup.foo, GroupNode)
# Overwrite a leaf node
new_path = os.path.join(mydir, 'data/nuts.csv')
package1._set(['newgroup', 'foo'], 'data/nuts.csv', build_dir=mydir)
assert package1.newgroup.foo._data() == new_path
# Overwrite the whole group
package1._set(['newgroup'], 'data/nuts.csv', build_dir=mydir)
assert package1.newgroup._data() == new_path
# Built a new package and verify the new contents
command.build('foo/package3', package1)
from quilt.data.foo import package3
assert hasattr(package3, 'dataframes2')
assert not hasattr(package3, 'dataframes')
new_csv = package3.dataframes2.csv._data()
assert new_csv.xs(0)['Int0'] == 42
new_df = package3.new.df._data()
assert new_df.xs(2)['a'] == 3
new_file = package3.new.file._data()
assert isinstance(new_file, string_types)
def find_package(self):
parts = self.package_name.split('/')
store, package = PackageStore.find_package(None, parts[0], parts[1])
return store, package
def test_save(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package1', build_path)
from quilt.data.foo import package1
# Build an identical package
command.build('foo/package2', package1)
from quilt.data.foo import package2
teststore = PackageStore(self._store_dir)
contents1 = teststore.get_package(None, 'foo', 'package1')
contents2 = teststore.get_package(None, 'foo', 'package2')
assert contents1 == contents2
# Rename an attribute
package1.dataframes2 = package1.dataframes
del package1.dataframes
# Modify an existing dataframe
csv = package1.dataframes2.csv._data()
csv.at[0, 'Int0'] = 42
# Add a new dataframe
df = pd.DataFrame(dict(a=[1, 2, 3]))
package1._set(['new', 'df'], df)
assert package1.new.df._data() is df
# Add some ndarrays
arr1 = np.array([[[1], [2], [3]], [[4], [5], [6]]])
arr2 = np.random.rand(30, 40, 50)
package1._set(['new', 'array1'], arr1)
package1._set(['new', 'array2'], arr2)
assert package1.new.array1._data() is arr1
assert package1.new.array2._data() is arr2
# Add a new file
file_path = os.path.join(mydir, 'data/foo.csv')
package1._set(['new', 'file'], 'data/foo.csv', build_dir=mydir)
assert package1.new.file._data() == file_path
# Add a new group
package1._add_group('newgroup')
assert isinstance(package1.newgroup, GroupNode)
package1.newgroup._add_group('foo')
assert isinstance(package1.newgroup.foo, GroupNode)
# Overwrite a leaf node
new_path = os.path.join(mydir, 'data/nuts.csv')
package1._set(['newgroup', 'foo'], 'data/nuts.csv', build_dir=mydir)
assert package1.newgroup.foo._data() == new_path
# Overwrite the whole group
package1._set(['newgroup'], 'data/nuts.csv', build_dir=mydir)
assert package1.newgroup._data() == new_path
# Set some custom metadata
package1._meta['foo'] = 'bar'
package1.newgroup._meta['x'] = 'y'
# Built a new package and verify the new contents
command.build('foo/package3', package1)
from quilt.data.foo import package3
assert hasattr(package3, 'dataframes2')
assert not hasattr(package3, 'dataframes')
new_csv = package3.dataframes2.csv._data()
assert new_csv.xs(0)['Int0'] == 42
new_df = package3.new.df._data()
assert new_df.xs(2)['a'] == 3
new_arr1 = package3.new.array1._data()
new_arr2 = package3.new.array2._data()
assert new_arr1[1][2][0] == 6
assert new_arr2.shape == (30, 40, 50)
new_file = package3.new.file._data()
assert isinstance(new_file, string_types)
assert package3._meta['foo'] == 'bar'
assert package3.newgroup._meta['x'] == 'y'
# Try setting invalid metadata
package1.new.df._meta['_system'] = 1
with self.assertRaises(command.CommandException):
command.build('foo/package4', package1)
package3._meta['foo'] = {'bar': lambda x: x}
with self.assertRaises(command.CommandException):
command.build('foo/package5', package3)