def test_load_update(self): # also tests dynamic import mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, './build.yml') command.build('foo/package5', build_path) from ..data.foo import package5 # make a copy, to prove we can newpkgname = 'foo/copied_package' command.build(newpkgname, package5) newfilename = 'myfile' + str(int(time.time())) with open(newfilename, 'w') as fh: fh.write('hello world1') module = command.load(newpkgname) module._set([newfilename], newfilename) command.build(newpkgname, module) # current spec requires that build() *not* update the in-memory module tree. newpath1 = getattr(module, newfilename)() assert newpath1 == newfilename # current spec requires that load() reload from disk, i.e. gets a reference # to the local object store # this is important because of potential changes to myfile reloaded_module = command.load(newpkgname) assert reloaded_module is not module newpath2 = getattr(reloaded_module, newfilename)() assert 'myfile' not in newpath2
def test_load_by_hash(self): """ Tests loading two different versions of the same package using command.load and specifying the package hash. """ # Old Version mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, './build.yml') command.build('foo/package', build_path) package = command.load('foo/package') pkghash = package._package.get_hash() # New Version mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, './build_simple.yml') command.build('foo/package', build_path) command.ls() load_pkg_new = command.load('foo/package') load_pkg_old = command.load('foo/package', hash=pkghash) assert load_pkg_old._package.get_hash() == pkghash assert load_pkg_new.foo with self.assertRaises(AttributeError): load_pkg_new.dataframes # Known failure cases # At present load does not support extended package syntax with self.assertRaises(command.CommandException): command.load('foo/package:h:%s' % pkghash) with self.assertRaises(command.CommandException): command.load('foo/package:t:latest') with self.assertRaises(command.CommandException): command.load('foo/package:v:1.0.0')
def test_asa_plot_formats_output(self): from PIL import Image from matplotlib import pyplot as plt from quilt.asa.img import plot mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, 'build_img.yml') command.build('foo/imgtest', build_path) pkg = command.load('foo/imgtest') outfile = os.path.join('.', 'temp-formats-plot.png') # pylint: disable=no-member pkg.mixed.img(asa=plot(figsize=(10, 10), formats=['png'])) # size * dpi = 1000 x 1000 pixels plt.savefig(outfile, dpi=100, format='png', transparent=False) ref_path = os.path.join(mydir, 'data', 'ref-asa-formats.png') ref_img = Image.open(ref_path) tst_img = Image.open(outfile) assert self._are_similar(ref_img, tst_img), \ 'render differs from reference: {}'.format(ref_img)
def test_filtering(self): mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, './build.yml') command.build('foo/package', build_path) pkg = command.load('foo/package') pkg.dataframes._meta['foo'] = 'bar' # "False" filter empty = pkg._filter(lambda node, name: False) assert not empty._keys() # "True" filter pkg_copy = pkg._filter(lambda node, name: True) assert set(pkg_copy._keys()) == set(pkg._keys()) assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys()) # Group nodes are copied. assert pkg_copy is not pkg assert pkg_copy.dataframes is not pkg.dataframes # Group metadata is a copy of the original. assert pkg_copy.dataframes._meta is not pkg.dataframes._meta assert pkg_copy.dataframes._meta == pkg.dataframes._meta # Leaf nodes are the originals. assert pkg_copy.README is pkg.README assert pkg_copy.dataframes.csv is pkg.dataframes.csv # "True" using dict syntax. pkg_copy = pkg._filter({}) assert set(pkg_copy._keys()) == set(pkg._keys()) # Non-existant metadata. pkg_copy = pkg._filter({'meta': {'non_existant': 'blah'}}) assert not pkg_copy._keys() # Single node. pkg_copy = pkg._filter({'name': 'csv'}) assert set(pkg_copy._keys()) == {'dataframes'} assert set(pkg_copy.dataframes._keys()) == {'csv'} # Returning "True" for a group copies its children. pkg_copy = pkg._filter({'meta': {'foo': 'bar'}}) assert set(pkg_copy._keys()) == {'dataframes'} assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys()) # Same thing for the root node. pkg_copy = pkg._filter({'name': ''}) assert set(pkg_copy._keys()) == set(pkg._keys()) # System metadata. pkg_copy = pkg._filter({'meta': {'_system': {'transform': 'csv'}}}) assert set(pkg_copy._keys()) == {'dataframes'} assert set(pkg_copy.dataframes._keys()) == set(pkg.dataframes._keys()) # Invalid filter. with self.assertRaises(ValueError): pkg._filter([]) with self.assertRaises(ValueError): pkg._filter({'whatever': 'blah'})
def test_memory_only_datanode_asa(self): testdata = "justatest" def test_lambda(node, hashes): return testdata mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, './build.yml') command.build('foo/package', build_path) pkg = command.load('foo/package') pkg._set(['dataframes', 'memory'], pd.DataFrame()) with self.assertRaises(ValueError): assert pkg.dataframes.memory(asa=test_lambda) is testdata
def test_datanode_asa(self): testdata = "justatest" def test_lambda(node, hashes): assert isinstance(node, DataNode) assert hashes for path in hashes: assert os.path.exists(path) return testdata mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, './build.yml') command.build('foo/package', build_path) pkg = command.load('foo/package') assert pkg.dataframes.csv(asa=test_lambda) is testdata
def test_asa_plot(self): from quilt.asa.img import plot mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, './build_img.yml') command.build('foo/imgtest', build_path) pkg = command.load('foo/imgtest') # expect no exceptions on root pkg(asa=plot()) # pylint: disable=no-member # expect no exceptions on GroupNode with only DF children pkg.dataframes(asa=plot()) # expect no exceptions on GroupNode with mixed children pkg.mixed(asa=plot()) # expect no exceptions on dir of images pkg.mixed.img(asa=plot()) pkg.mixed.img(asa=plot(formats=['jpg', 'png'])) # assert images != filtered, 'Expected only .jpg and .png images' # expect no exceptions on single images pkg.mixed.img.sf(asa=plot()) pkg.mixed.img.portal(asa=plot())
def test_asa_pytorch(self): """test asa.torch interface by converting a GroupNode with asa=""" from torchvision.transforms import Compose, CenterCrop, ToTensor, Resize from torch.utils.data import Dataset from PIL import Image from torch import Tensor from quilt.asa.pytorch import dataset # pylint: disable=missing-docstring # helper functions to simulate real pytorch dataset usage def calculate_valid_crop_size(crop_size, upscale_factor): return crop_size - (crop_size % upscale_factor) def node_parser(node): path = node() if isinstance(path, string_types): img = Image.open(path).convert('YCbCr') chan, _, _ = img.split() return chan else: raise TypeError('Expected string path to an image fragment') def input_transform(crop_size, upscale_factor): return Compose([ CenterCrop(crop_size), Resize(crop_size // upscale_factor), ToTensor(), ]) def target_transform(crop_size): def _inner(img): img_ = img.copy() return Compose([ CenterCrop(crop_size), ToTensor(), ])(img_) return _inner # pylint: disable=protected-access def is_image(node): """file extension introspection on Quilt nodes""" if isinstance(node, DataNode): filepath = node._meta.get('_system', {}).get('filepath') if filepath: return any( filepath.endswith(extension) for extension in [".png", ".jpg", ".jpeg"]) # end helper functions mydir = os.path.dirname(__file__) build_path = os.path.join(mydir, 'build_img.yml') command.build('foo/torchtest', build_path) pkg = command.load('foo/torchtest') upscale_factor = 3 crop_size = calculate_valid_crop_size(256, upscale_factor) # pylint: disable=no-member my_dataset = pkg.mixed.img(asa=dataset( include=is_image, node_parser=node_parser, input_transform=input_transform(crop_size, upscale_factor), target_transform=target_transform(crop_size) )) assert isinstance(my_dataset, Dataset), \ 'expected type {}, got {}'.format(type(Dataset), type(my_dataset)) assert my_dataset.__len__() == 2, \ 'expected two images in mixed.img, got {}'.format(my_dataset.__len__()) for i in range(my_dataset.__len__()): tens = my_dataset.__getitem__(i) assert all((isinstance(x, Tensor) for x in tens)), \ 'Expected all torch.Tensors in tuple, got {}'.format(tens)