def test_load_update(self):
# also tests dynamic import
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package5', build_path)
from ..data.foo import package5
# make a copy, to prove we can
newpkgname = 'foo/copied_package'
command.build(newpkgname, package5)
newfilename = 'myfile' + str(int(time.time()))
with open(newfilename, 'w') as fh:
fh.write('hello world1')
module = command.load(newpkgname)
module._set([newfilename], newfilename)
command.build(newpkgname, module)
# current spec requires that build() *not* update the in-memory module tree.
newpath1 = getattr(module, newfilename)()
assert newpath1 == newfilename
# current spec requires that load() reload from disk, i.e. gets a reference
# to the local object store
# this is important because of potential changes to myfile
reloaded_module = command.load(newpkgname)
assert reloaded_module is not module
newpath2 = getattr(reloaded_module, newfilename)()
assert 'myfile' not in newpath2
def test_load_by_hash(self):
"""
Tests loading two different versions of the same
package using command.load and specifying the package
hash.
"""
# Old Version
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
package = command.load('foo/package')
pkghash = package._package.get_hash()
# New Version
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_simple.yml')
command.build('foo/package', build_path)
command.ls()
load_pkg_new = command.load('foo/package')
load_pkg_old = command.load('foo/package', hash=pkghash)
assert load_pkg_old._package.get_hash() == pkghash
assert load_pkg_new.foo
with self.assertRaises(AttributeError):
load_pkg_new.dataframes
# Known failure cases
# At present load does not support extended package syntax
with self.assertRaises(command.CommandException):
command.load('foo/package:h:%s' % pkghash)
with self.assertRaises(command.CommandException):
command.load('foo/package:t:latest')
with self.assertRaises(command.CommandException):
command.load('foo/package:v:1.0.0')
def test_asa_plot_formats_output(self):
from PIL import Image
from matplotlib import pyplot as plt
from quilt.asa.img import plot
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, 'build_img.yml')
command.build('foo/imgtest', build_path)
pkg = command.load('foo/imgtest')
outfile = os.path.join('.', 'temp-formats-plot.png')
# pylint: disable=no-member
pkg.mixed.img(asa=plot(figsize=(10, 10), formats=['png']))
# size * dpi = 1000 x 1000 pixels
plt.savefig(outfile, dpi=100, format='png', transparent=False)
ref_path = os.path.join(mydir, 'data', 'ref-asa-formats.png')
ref_img = Image.open(ref_path)
tst_img = Image.open(outfile)
assert self._are_similar(ref_img, tst_img), \
'render differs from reference: {}'.format(ref_img)
def test_filtering(self):
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
pkg = command.load('foo/package')
pkg.dataframes._meta['foo'] = 'bar'
# "False" filter
empty = pkg._filter(lambda node, name: False)
assert not empty._keys()
# "True" filter
pkg_copy = pkg._filter(lambda node, name: True)
assert set(pkg_copy._keys()) == set(pkg._keys())
assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys())
# Group nodes are copied.
assert pkg_copy is not pkg
assert pkg_copy.dataframes is not pkg.dataframes
# Group metadata is a copy of the original.
assert pkg_copy.dataframes._meta is not pkg.dataframes._meta
assert pkg_copy.dataframes._meta == pkg.dataframes._meta
# Leaf nodes are the originals.
assert pkg_copy.README is pkg.README
assert pkg_copy.dataframes.csv is pkg.dataframes.csv
# "True" using dict syntax.
pkg_copy = pkg._filter({})
assert set(pkg_copy._keys()) == set(pkg._keys())
# Non-existant metadata.
pkg_copy = pkg._filter({'meta': {'non_existant': 'blah'}})
assert not pkg_copy._keys()
# Single node.
pkg_copy = pkg._filter({'name': 'csv'})
assert set(pkg_copy._keys()) == {'dataframes'}
assert set(pkg_copy.dataframes._keys()) == {'csv'}
# Returning "True" for a group copies its children.
pkg_copy = pkg._filter({'meta': {'foo': 'bar'}})
assert set(pkg_copy._keys()) == {'dataframes'}
assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys())
# Same thing for the root node.
pkg_copy = pkg._filter({'name': ''})
assert set(pkg_copy._keys()) == set(pkg._keys())
# System metadata.
pkg_copy = pkg._filter({'meta': {'_system': {'transform': 'csv'}}})
assert set(pkg_copy._keys()) == {'dataframes'}
assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys())
# Invalid filter.
with self.assertRaises(ValueError):
pkg._filter([])
with self.assertRaises(ValueError):
pkg._filter({'whatever': 'blah'})
def test_memory_only_datanode_asa(self):
testdata = "justatest"
def test_lambda(node, hashes):
return testdata
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
pkg = command.load('foo/package')
pkg._set(['dataframes', 'memory'], pd.DataFrame())
with self.assertRaises(ValueError):
assert pkg.dataframes.memory(asa=test_lambda) is testdata
def test_datanode_asa(self):
testdata = "justatest"
def test_lambda(node, hashes):
assert isinstance(node, DataNode)
assert hashes
for path in hashes:
assert os.path.exists(path)
return testdata
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build.yml')
command.build('foo/package', build_path)
pkg = command.load('foo/package')
assert pkg.dataframes.csv(asa=test_lambda) is testdata
def test_asa_plot(self):
from quilt.asa.img import plot
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, './build_img.yml')
command.build('foo/imgtest', build_path)
pkg = command.load('foo/imgtest')
# expect no exceptions on root
pkg(asa=plot())
# pylint: disable=no-member
# expect no exceptions on GroupNode with only DF children
pkg.dataframes(asa=plot())
# expect no exceptions on GroupNode with mixed children
pkg.mixed(asa=plot())
# expect no exceptions on dir of images
pkg.mixed.img(asa=plot())
pkg.mixed.img(asa=plot(formats=['jpg', 'png']))
# assert images != filtered, 'Expected only .jpg and .png images'
# expect no exceptions on single images
pkg.mixed.img.sf(asa=plot())
pkg.mixed.img.portal(asa=plot())
def test_asa_pytorch(self):
"""test asa.torch interface by converting a GroupNode with asa="""
from torchvision.transforms import Compose, CenterCrop, ToTensor, Resize
from torch.utils.data import Dataset
from PIL import Image
from torch import Tensor
from quilt.asa.pytorch import dataset
# pylint: disable=missing-docstring
# helper functions to simulate real pytorch dataset usage
def calculate_valid_crop_size(crop_size, upscale_factor):
return crop_size - (crop_size % upscale_factor)
def node_parser(node):
path = node()
if isinstance(path, string_types):
img = Image.open(path).convert('YCbCr')
chan, _, _ = img.split()
return chan
else:
raise TypeError('Expected string path to an image fragment')
def input_transform(crop_size, upscale_factor):
return Compose([
CenterCrop(crop_size),
Resize(crop_size // upscale_factor),
ToTensor(),
])
def target_transform(crop_size):
def _inner(img):
img_ = img.copy()
return Compose([
CenterCrop(crop_size),
ToTensor(),
])(img_)
return _inner
# pylint: disable=protected-access
def is_image(node):
"""file extension introspection on Quilt nodes"""
if isinstance(node, DataNode):
filepath = node._meta.get('_system', {}).get('filepath')
if filepath:
return any(
filepath.endswith(extension)
for extension in [".png", ".jpg", ".jpeg"])
# end helper functions
mydir = os.path.dirname(__file__)
build_path = os.path.join(mydir, 'build_img.yml')
command.build('foo/torchtest', build_path)
pkg = command.load('foo/torchtest')
upscale_factor = 3
crop_size = calculate_valid_crop_size(256, upscale_factor)
# pylint: disable=no-member
my_dataset = pkg.mixed.img(asa=dataset(
include=is_image,
node_parser=node_parser,
input_transform=input_transform(crop_size, upscale_factor),
target_transform=target_transform(crop_size)
))
assert isinstance(my_dataset, Dataset), \
'expected type {}, got {}'.format(type(Dataset), type(my_dataset))
assert my_dataset.__len__() == 2, \
'expected two images in mixed.img, got {}'.format(my_dataset.__len__())
for i in range(my_dataset.__len__()):
tens = my_dataset.__getitem__(i)
assert all((isinstance(x, Tensor) for x in tens)), \
'Expected all torch.Tensors in tuple, got {}'.format(tens)
