def test_push_subpackage(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/bar', build_path)
_, pkgroot = store.PackageStore.find_package(None, 'foo', 'bar')
contents = pkgroot.children['dataframes']
all_hashes = set(find_object_hashes(contents))
upload_urls = {
blob_hash:
dict(head="https://example.com/head/{owner}/{hash}".format(
owner='foo', hash=blob_hash),
put="https://example.com/put/{owner}/{hash}".format(
owner='foo', hash=blob_hash))
for blob_hash in all_hashes
}
for blob_hash in all_hashes:
urls = upload_urls[blob_hash]
self.requests_mock.add(responses.HEAD, urls['head'], status=404)
self.requests_mock.add(responses.PUT, urls['put'])
self._mock_update_package('foo/bar', 'dataframes', contents,
upload_urls)
# Push a subpackage.
command.push('foo/bar/dataframes')
def test_load_by_hash(self):
"""
Tests loading two different versions of the same
package using command.load and specifying the package
hash.
"""
# Old Version
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
package = command.load('foo/package')
pkghash = package._package.get_hash()
# New Version
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/package', build_path)
command.ls()
load_pkg_new = command.load('foo/package')
load_pkg_old = command.load('foo/package', hash=pkghash)
assert load_pkg_old._package.get_hash() == pkghash
assert load_pkg_new.foo
with self.assertRaises(AttributeError):
load_pkg_new.dataframes
# Known failure cases
# At present load does not support extended package syntax
with self.assertRaises(command.CommandException):
command.load('foo/package:h:%s' % pkghash)
with self.assertRaises(command.CommandException):
command.load('foo/package:t:latest')
with self.assertRaises(command.CommandException):
command.load('foo/package:v:1.0.0')
def test_import_group_as_data(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_group_data.yml')
command.build('foo/grppkg', build_path)
# Good imports
from quilt.data.foo import grppkg
assert isinstance(grppkg.dataframes, GroupNode)
# Make sure child dataframes were concatenated in the correct order (alphabetically by node name).
df = grppkg.dataframes._data()
assert df['x'].tolist() == [1, 2, 3, 4]
assert df['y'].tolist() == [1, 4, 9, 16]
# Incompatible Schema
with self.assertRaises(StoreException):
grppkg.incompatible._data()
# Empty group
grppkg.dataframes._add_group("empty")
assert grppkg.dataframes.empty._data() is None
# In-memory dataframe
grppkg._set(['dataframes', 'foo'], pd.DataFrame([1, 2, 3]))
with self.assertRaises(NotImplementedError):
grppkg.dataframes._data()
def test_asa_plot_formats_output(self):
from PIL import Image
from matplotlib import pyplot as plt
from quilt.asa.img import plot
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, 'build_img.yml')
command.build('foo/imgtest', build_path)
pkg = command.load('foo/imgtest')
outfile = os.path.join('.', 'temp-formats-plot.png')
# pylint: disable=no-member
pkg.mixed.img(asa=plot(figsize=(10, 10), formats=['png']))
# size * dpi = 1000 x 1000 pixels
plt.savefig(outfile, dpi=100, format='png', transparent=False)
ref_path = os.path.join(mydir, 'data', 'ref-asa-formats.png')
ref_img = Image.open(ref_path)
tst_img = Image.open(outfile)
assert self._are_similar(ref_img, tst_img), \
'render differs from reference: {}'.format(ref_img)
def test_build_from_git_branch(self):
branch = 'notmaster'
git_url = 'https://github.com/quiltdata/testdata.git'
def mock_git_clone(cmd):
# test git command
assert len(cmd) == 8
assert cmd[:7] == [
'git', 'clone', '-q', '--depth=1', '-b', branch, git_url
]
# fake git clone by copying test files into destpath
srcfile = 'foo.csv'
mydir = os.path.dirname(__file__)
srcpath = os.path.join(mydir, 'data', srcfile)
destpath = os.path.join(cmd[-1], srcfile)
shutil.copyfile(srcpath, destpath)
with patch('subprocess.check_call', mock_git_clone):
command.build('user/test', "{url}@{brch}".format(url=git_url,
brch=branch))
from quilt.data.user import test
assert hasattr(test, 'foo')
assert isinstance(test.foo(), pd.DataFrame)
def test_filtering(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
pkg = command.load('foo/package')
pkg.dataframes._meta['foo'] = 'bar'
# "False" filter
empty = pkg._filter(lambda node, name: False)
assert not empty._keys()
# "True" filter
pkg_copy = pkg._filter(lambda node, name: True)
assert set(pkg_copy._keys()) == set(pkg._keys())
assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys())
# Group nodes are copied.
assert pkg_copy is not pkg
assert pkg_copy.dataframes is not pkg.dataframes
# Group metadata is a copy of the original.
assert pkg_copy.dataframes._meta is not pkg.dataframes._meta
assert pkg_copy.dataframes._meta == pkg.dataframes._meta
# Leaf nodes are the originals.
assert pkg_copy.README is pkg.README
assert pkg_copy.dataframes.csv is pkg.dataframes.csv
# "True" using dict syntax.
pkg_copy = pkg._filter({})
assert set(pkg_copy._keys()) == set(pkg._keys())
# Non-existant metadata.
pkg_copy = pkg._filter({'meta': {'non_existant': 'blah'}})
assert not pkg_copy._keys()
# Single node.
pkg_copy = pkg._filter({'name': 'csv'})
assert set(pkg_copy._keys()) == {'dataframes'}
assert set(pkg_copy.dataframes._keys()) == {'csv'}
# Returning "True" for a group copies its children.
pkg_copy = pkg._filter({'meta': {'foo': 'bar'}})
assert set(pkg_copy._keys()) == {'dataframes'}
assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys())
# Same thing for the root node.
pkg_copy = pkg._filter({'name': ''})
assert set(pkg_copy._keys()) == set(pkg._keys())
# System metadata.
pkg_copy = pkg._filter({'meta': {'_system': {'transform': 'csv'}}})
assert set(pkg_copy._keys()) == {'dataframes'}
assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys())
# Invalid filter.
with self.assertRaises(ValueError):
pkg._filter([])
with self.assertRaises(ValueError):
pkg._filter({'whatever': 'blah'})
def test_team_imports(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('test:bar/package', build_path)
# Good imports
from quilt.team.test.bar import package
from quilt.team.test.bar.package import dataframes
from quilt.team.test.bar.package import README
# Contents of the imports
assert isinstance(package, GroupNode)
assert isinstance(dataframes, GroupNode)
assert isinstance(dataframes.csv, DataNode)
assert isinstance(README, DataNode)
assert package.dataframes == dataframes
assert package.README == README
assert set(dataframes._keys()) == {'csv', 'nulls'}
assert set(dataframes._group_keys()) == set()
assert set(dataframes._data_keys()) == {'csv', 'nulls'}
assert isinstance(README(), string_types)
assert isinstance(README._data(), string_types)
assert isinstance(dataframes.csv(), pd.DataFrame)
assert isinstance(dataframes.csv._data(), pd.DataFrame)
str(package)
str(dataframes)
str(README)
# Bad attributes of imported packages
with self.assertRaises(AttributeError):
package.foo
with self.assertRaises(AttributeError):
package.dataframes.foo
with self.assertRaises(AttributeError):
package.dataframes.csv.foo
# Bad imports
with self.assertRaises(ImportError):
import quilt.team.test.bar.bad_package
with self.assertRaises(ImportError):
import quilt.team.test.bad_user.bad_package
with self.assertRaises(ImportError):
from quilt.team.test.bar.dataframes import blah
with self.assertRaises(ImportError):
from quilt.team.test.bar.baz import blah
def test_rm(self):
"""
Test removing a package.
"""
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/bar', build_path)
command.rm('foo/bar', force=True)
teststore = store.PackageStore(self._store_dir)
assert not os.path.isdir(teststore.package_path('foo', 'bar'))
def test_log(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
owner = 'foo'
package = 'bar'
command.build('%s/%s' % (owner, package), build_path)
pkg_obj = store.PackageStore.find_package(owner, package)
self._mock_logs_list(owner, package, pkg_obj.get_hash())
command.log("{owner}/{pkg}".format(owner=owner, pkg=package))
def test_team_set_non_node_attr(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('test:bar/package4', build_path)
from quilt.team.test.bar import package4
# Assign a DataFrame as a node
# (should throw exception)
df = pd.DataFrame(dict(a=[1, 2, 3]))
with self.assertRaises(AttributeError):
package4.newdf = df
def test_memory_only_datanode_asa(self):
testdata = "justatest"
def test_lambda(node, hashes):
return testdata
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
pkg = command.load('foo/package')
pkg._set(['dataframes', 'memory'], pd.DataFrame())
with self.assertRaises(ValueError):
assert pkg.dataframes.memory(asa=test_lambda) is testdata
def test_push(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/bar', build_path)
_, pkgroot = store.PackageStore.find_package(None, 'foo', 'bar')
pkg_hash = hash_contents(pkgroot)
assert pkg_hash
contents = pkgroot
all_hashes = set(find_object_hashes(contents))
upload_urls = {
blob_hash:
dict(head="https://example.com/head/{owner}/{hash}".format(
owner='foo', hash=blob_hash),
put="https://example.com/put/{owner}/{hash}".format(
owner='foo', hash=blob_hash))
for blob_hash in all_hashes
}
# We will push the package 3 times, so we're mocking all responses 3 times.
for blob_hash in all_hashes:
urls = upload_urls[blob_hash]
# First time the package is pushed, s3 HEAD 404s, and we get a PUT.
self.requests_mock.add(responses.HEAD, urls['head'], status=404)
self.requests_mock.add(responses.PUT, urls['put'])
# Second and third times, s3 HEAD succeeds, and we're not expecting a PUT.
self.requests_mock.add(responses.HEAD, urls['head'])
self.requests_mock.add(responses.HEAD, urls['head'])
self._mock_put_package('foo/bar', pkg_hash, contents, upload_urls)
self._mock_put_tag('foo/bar', 'latest')
self._mock_put_package('foo/bar', pkg_hash, contents, upload_urls)
self._mock_put_tag('foo/bar', 'latest')
self._mock_put_package('foo/bar', pkg_hash, contents, upload_urls)
self._mock_put_tag('foo/bar', 'latest')
# Push a new package.
command.push('foo/bar')
# Push it again; this time, we're verifying that there are no s3 uploads.
command.push('foo/bar')
# Push the package by its hash
command.push('foo/bar', hash=pkg_hash)
def test_team_non_team_imports(self):
mydir = os.path.dirname(__file__)
build_path1 = os.path.join(mydir, './build_simple.yml')
command.build('myteam:foo/team_imports', build_path1)
build_path2 = os.path.join(mydir, './build_empty.yml')
command.build('foo/team_imports', build_path2)
# Verify that both imports work, and packages are in fact different.
from ..team.myteam.foo import team_imports as pkg1
from ..data.foo import team_imports as pkg2
assert hasattr(pkg1, 'foo')
assert not hasattr(pkg2, 'foo')
def test_import_group_as_data(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_group_data.yml')
command.build('foo/grppkg', build_path)
# Good imports
from quilt.data.foo.grppkg import dataframes
assert isinstance(dataframes, GroupNode)
assert isinstance(dataframes.csvs.csv, DataNode)
assert isinstance(dataframes._data(), pd.DataFrame)
# Incompatible Schema
from quilt.data.foo.grppkg import incompatible
with self.assertRaises(PackageException):
incompatible._data()
def test_datanode_asa(self):
testdata = "justatest"
def test_lambda(node, hashes):
assert isinstance(node, DataNode)
assert hashes
for path in hashes:
assert os.path.exists(path)
return testdata
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
pkg = command.load('foo/package')
assert pkg.dataframes.csv(asa=test_lambda) is testdata
def test_rm_package_w_shared_obj(self):
"""
Test removing a package that shares an object with another. The
other package should still remain.
"""
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/bar', build_path)
command.build('foo/bar2', build_path)
command.rm('foo/bar', force=True)
teststore = store.PackageStore(self._store_dir)
assert not os.path.isdir(teststore.package_path('foo', 'bar'))
from quilt.data.foo import bar2
assert isinstance(bar2.foo(), pd.DataFrame)
def test_build_checks_yaml_syntax_error(self): # pylint: disable=C0103
path = os.path.abspath(os.path.dirname(__file__))
buildfilepath = os.path.join(path, 'build_checks_bad_syntax.yml')
checksorigpath = os.path.join(path, 'checks_bad_syntax.yml')
checksfilepath = os.path.join(path, 'checks.yml')
try:
origdir = os.curdir
os.chdir(path)
assert not os.path.exists(checksfilepath)
shutil.copy(checksorigpath, checksfilepath)
with assertRaisesRegex(self, command.CommandException,
r'Bad yaml syntax.*checks\.yml'):
command.build('user/test', buildfilepath)
finally:
os.remove(checksfilepath)
os.chdir(origdir)
def test_push(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/bar', build_path)
pkg_obj = store.PackageStore.find_package('foo', 'bar')
pkg_hash = pkg_obj.get_hash()
assert pkg_hash
contents = pkg_obj.get_contents()
urls = upload_urls(contents)
for url in urls.values():
self._mock_s3(url)
self._mock_put_package('foo/bar', pkg_hash, contents)
self._mock_put_tag('foo/bar', 'latest')
command.push('foo/bar')
def test_import_group_as_data(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_group_data.yml')
command.build('foo/grppkg', build_path)
# Good imports
from quilt.data.foo.grppkg import dataframes
assert isinstance(dataframes, GroupNode)
# Make sure child dataframes were concatenated in the correct order (alphabetically by node name).
df = dataframes._data()
assert df['x'].tolist() == [1, 2, 3, 4]
assert df['y'].tolist() == [1, 4, 9, 16]
# Incompatible Schema
from quilt.data.foo.grppkg import incompatible
with self.assertRaises(PackageException):
incompatible._data()
def test_git_clone_fail(self):
git_url = 'https://github.com/quiltdata/testdata.git'
def mock_git_clone(cmd):
# test git command
assert len(cmd) == 6
assert cmd[:5] == ['git', 'clone', '-q', '--depth=1', git_url]
# fake git clone fail
raise Exception()
with patch('subprocess.check_call', mock_git_clone):
with self.assertRaises(command.CommandException):
command.build('user/test', git_url)
from quilt.data.user import test
assert hasattr(test, 'foo')
assert isinstance(test.foo(), pd.DataFrame)
def test_multiple_package_dirs(self):
# First level
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/nested', build_path)
# Second level: different package
os.mkdir("aaa")
os.chdir("aaa")
build_path = os.path.join(mydir, './build.yml')
command.build('foo/nested', build_path)
# Third level: empty package directory
os.mkdir("bbb")
os.chdir("bbb")
os.mkdir(PACKAGE_DIR_NAME)
# Imports should find the second package
from quilt.data.foo.nested import dataframes
def test_rm_doesnt_break_cache(self):
"""
Test building, removing then rebuilding a package. The package
should be correctly rebuilt.
"""
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/bar', build_path)
command.rm('foo/bar', force=True)
teststore = store.PackageStore(self._store_dir)
assert not os.path.isdir(teststore.package_path('foo', 'bar'))
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/bar', build_path)
from quilt.data.foo import bar
assert isinstance(bar.foo(), pd.DataFrame)
def test_multiple_updates(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package6', build_path)
from ..data.foo import package6
newfilename1 = 'myfile1' + str(int(time.time()))
with open(newfilename1, 'w') as fh:
fh.write('hello world1')
package6._set([newfilename1], newfilename1)
newfilename2 = 'myfile2' + str(int(time.time()))
with open(newfilename2, 'w') as fh:
fh.write('hello world2')
package6._set([newfilename1], newfilename2)
assert getattr(package6, newfilename1)() == newfilename2
def test_asa_plot(self):
from quilt.asa.img import plot
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_img.yml')
command.build('foo/imgtest', build_path)
pkg = command.load('foo/imgtest')
# expect no exceptions on root
pkg(asa=plot())
# pylint: disable=no-member
# expect no exceptions on GroupNode with only DF children
pkg.dataframes(asa=plot())
# expect no exceptions on GroupNode with mixed children
pkg.mixed(asa=plot())
# expect no exceptions on dir of images
pkg.mixed.img(asa=plot())
pkg.mixed.img(asa=plot(formats=['jpg', 'png']))
# assert images != filtered, 'Expected only .jpg and .png images'
# expect no exceptions on single images
pkg.mixed.img.sf(asa=plot())
pkg.mixed.img.portal(asa=plot())
def test_git_clone_fail(self):
git_url = 'https://github.com/quiltdata/testdata.git'
def mock_git_clone(cmd):
# test git command
assert len(cmd) == 6
assert cmd[:5] == ['git', 'clone', '-q', '--depth=1', git_url]
# fake git clone fail
raise Exception()
with patch('subprocess.check_call', mock_git_clone):
with self.assertRaises(command.CommandException):
command.build('user/pkg__test_git_clone_fail', git_url)
# TODO: running -n (pytest-xdist) there's leaky state and can throw
# either ImportError: cannot import name or ModuleNotFoundError
with assertRaisesRegex(
self, Exception,
r'cannot import|not found|No module named|Could not find'):
from quilt.data.user import pkg__test_git_clone_fail
def test_load_update(self):
# also tests dynamic import
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package5', build_path)
from ..data.foo import package5
# make a copy, to prove we can
newpkgname = 'foo/copied_package'
command.build(newpkgname, package5)
newfilename = 'myfile' + str(int(time.time()))
with open(newfilename, 'w') as fh:
fh.write('hello world1')
module = command.load(newpkgname)
module._set([newfilename], newfilename)
command.build(newpkgname, module)
# current spec requires that build() *not* update the in-memory module tree.
newpath1 = getattr(module, newfilename)()
assert newpath1 == newfilename
# current spec requires that load() reload from disk, i.e. gets a reference
# to the local object store
# this is important because of potential changes to myfile
reloaded_module = command.load(newpkgname)
assert reloaded_module is not module
newpath2 = getattr(reloaded_module, newfilename)()
assert 'myfile' not in newpath2
def test_multiple_package_dirs(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir,
'./build.yml') # Contains 'dataframes'
simple_build_path = os.path.join(mydir, './build_simple.yml') # Empty
new_build_dir = 'aaa/bbb/%s' % PACKAGE_DIR_NAME
# Build two packages:
# - First one exists in the default dir and the new dir; default should take priority.
# - Second one only exists in the new dir.
# First package.
command.build('foo/multiple1', build_path)
# First and second package in the new build dir.
with patch.dict(os.environ,
{'QUILT_PRIMARY_PACKAGE_DIR': new_build_dir}):
command.build('foo/multiple1', simple_build_path)
command.build('foo/multiple2', simple_build_path)
# Cannot see the second package yet.
with self.assertRaises(ImportError):
from quilt.data.foo import multiple2
# Now search the new build dir.
dirs = 'foo/%s:%s' % (PACKAGE_DIR_NAME, new_build_dir)
with patch.dict(os.environ, {'QUILT_PACKAGE_DIRS': dirs}):
# Can import the second package now.
from quilt.data.foo import multiple2
# The first package contains data from the default dir.
from quilt.data.foo import multiple1
assert multiple1.dataframes
def test_logging(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
log_url = '%s/api/log' % (command.get_registry_url(), )
# Successful logging response.
with patch('quilt.tools.command._load_config', return_value={}):
def callback(request):
data = json.loads(request.body)
assert data == [
dict(
type='build',
package=hashlib.md5(b'foo/bar').hexdigest(),
dry_run=False,
env='default',
)
]
return (200, {}, '')
self.requests_mock.add_callback(responses.POST, log_url, callback)
command.build('foo/bar', build_path)
# Failed logging response.
with patch('quilt.tools.command._load_config', return_value={}):
self.requests_mock.add(responses.POST, log_url, status=500)
command.build('foo/bar', build_path)
# ConnectionError
with patch('quilt.tools.command._load_config', return_value={}):
self.requests_mock.add(responses.POST,
log_url,
body=requests.exceptions.ConnectionError())
command.build('foo/bar', build_path)
# Disabled logging.
with patch('quilt.tools.command._load_config',
return_value={'disable_analytics': True}):
self.requests_mock.add(responses.POST,
log_url,
body=AssertionError('Unexpected logging!'))
command.build('foo/bar', build_path)
self.requests_mock.reset(
) # Prevent the "not all requests ..." assert.
def test_save(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package1', build_path)
from quilt.data.foo import package1
# Build an identical package
command.build('foo/package2', package1)
contents1 = open('quilt_packages/foo/package1.json').read()
contents2 = open('quilt_packages/foo/package2.json').read()
assert contents1 == contents2
# Rename an attribute
package1.dataframes2 = package1.dataframes
del package1.dataframes
# Modify an existing dataframe
csv = package1.dataframes2.csv._data()
csv.set_value(0, 'Int0', 42)
# Add a new dataframe
df = pd.DataFrame(dict(a=[1, 2, 3]))
package1._set(['new', 'df'], df)
# Add a new file
file_path = os.path.join(mydir, 'data/foo.csv')
package1._set(['new', 'file'], file_path)
# Add a new group
package1._add_group("newgroup")
package1._set(['new', 'newgroup', 'df'], df)
# Can't overwrite things
with self.assertRaises(ValueError):
package1._set(['new'], file_path)
with self.assertRaises(ValueError):
package1._set(['new', 'file'], file_path)
# Built a new package and verify the new contents
command.build('foo/package3', package1)
from quilt.data.foo import package3
assert hasattr(package3, 'dataframes2')
assert not hasattr(package3, 'dataframes')
new_csv = package3.dataframes2.csv._data()
assert new_csv.xs(0)['Int0'] == 42
new_df = package3.new.df._data()
assert new_df.xs(2)['a'] == 3
new_file = package3.new.file._data()
assert isinstance(new_file, string_types)
def test_save(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package1', build_path)
from quilt.data.foo import package1
# Build an identical package
command.build('foo/package2', package1)
from quilt.data.foo import package2
teststore = PackageStore(self._store_dir)
contents1 = open(
os.path.join(teststore.package_path(None, 'foo', 'package1'),
Package.CONTENTS_DIR,
package1._package.get_hash())).read()
contents2 = open(
os.path.join(teststore.package_path(None, 'foo', 'package2'),
Package.CONTENTS_DIR,
package2._package.get_hash())).read()
assert contents1 == contents2
# Rename an attribute
package1.dataframes2 = package1.dataframes
del package1.dataframes
# Modify an existing dataframe
csv = package1.dataframes2.csv._data()
csv.at[0, 'Int0'] = 42
# Add a new dataframe
df = pd.DataFrame(dict(a=[1, 2, 3]))
package1._set(['new', 'df'], df)
assert package1.new.df._data() is df
# Add a new file
file_path = os.path.join(mydir, 'data/foo.csv')
package1._set(['new', 'file'], 'data/foo.csv', build_dir=mydir)
assert package1.new.file._data() == file_path
# Add a new group
package1._add_group('newgroup')
assert isinstance(package1.newgroup, GroupNode)
package1.newgroup._add_group('foo')
assert isinstance(package1.newgroup.foo, GroupNode)
# Overwrite a leaf node
new_path = os.path.join(mydir, 'data/nuts.csv')
package1._set(['newgroup', 'foo'], 'data/nuts.csv', build_dir=mydir)
assert package1.newgroup.foo._data() == new_path
# Overwrite the whole group
package1._set(['newgroup'], 'data/nuts.csv', build_dir=mydir)
assert package1.newgroup._data() == new_path
# Built a new package and verify the new contents
command.build('foo/package3', package1)
from quilt.data.foo import package3
assert hasattr(package3, 'dataframes2')
assert not hasattr(package3, 'dataframes')
new_csv = package3.dataframes2.csv._data()
assert new_csv.xs(0)['Int0'] == 42
new_df = package3.new.df._data()
assert new_df.xs(2)['a'] == 3
new_file = package3.new.file._data()
assert isinstance(new_file, string_types)
