def test_external_reference_invalid_fragment(tmpdir): exttree = { 'list_of_stuff': [ 'foobar', 42, np.array([7, 8, 9], np.float) ] } external_path = os.path.join(str(tmpdir), 'external.asdf') with asdf.AsdfFile(exttree) as ff: ff.write_to(external_path) tree = { 'foo': { '$ref': 'external.asdf#/list_of_stuff/a' } } with asdf.AsdfFile(tree, uri=pathname2url( os.path.join(str(tmpdir), 'main.asdf'))) as ff: with pytest.raises(ValueError): ff.resolve_references() tree = { 'foo': { '$ref': 'external.asdf#/list_of_stuff/3' } } with asdf.AsdfFile(tree, uri=pathname2url( os.path.join(str(tmpdir), 'main.asdf'))) as ff: with pytest.raises(ValueError): ff.resolve_references()
def get_read_fd(): f = generic_io.get_file(path, mode='r') assert isinstance(f, generic_io.RealFile) assert f._uri == urljoin('file:', pathname2url(path)) # This is to check for a "feature" in Python 3.x that reading zero # bytes from a socket causes it to stop. We have code in generic_io.py # to workaround it. f.read(0) return f
def get_read_fd(): f = generic_io.get_file(path, mode='r') assert isinstance(f, generic_io.RealFile) assert f._uri == urljoin('file:', pathname2url(path)) # This is to check for a "feature" in Python 3.x that reading zero # bytes from a socket causes it to stop. We have code in generic_io.py # to workaround it. f.read(0) return f
def __init__(self, fd, mode, close=True, uri=None): super(RealFile, self).__init__(fd, mode, close=close, uri=uri) stat = os.fstat(fd.fileno()) if sys.platform.startswith('win'): # There appears to be reliable way to get block size on Windows, # so just choose a reasonable default self._blksize = io.DEFAULT_BUFFER_SIZE else: self._blksize = stat.st_blksize self._size = stat.st_size if (uri is None and isinstance(fd.name, six.string_types) and os.path.exists(fd.name)): self._uri = urlparse.urljoin( 'file:', pathname2url(os.path.abspath(fd.name)))
def test_external_reference_invalid(tmpdir): tree = { 'foo': { '$ref': 'fail.asdf' } } ff = asdf.AsdfFile(tree) with pytest.raises(ValueError): ff.resolve_references() ff = asdf.AsdfFile(tree, uri="http://nowhere.com/") with pytest.raises(IOError): ff.resolve_references() ff = asdf.AsdfFile(tree, uri=pathname2url( os.path.join(str(tmpdir), 'main.asdf'))) with pytest.raises(IOError): ff.resolve_references()
def test_internal_reference(): tree = { 'foo': 2, 'bar': {'$ref': '#'} } ff = asdf.AsdfFile(tree) ff.find_references() assert isinstance(ff.tree['bar'], reference.Reference) ff.resolve_references() assert ff.tree['bar']['foo'] == 2 tree = { 'foo': 2 } ff = asdf.AsdfFile(tree, uri=pathname2url(os.path.abspath("test.asdf"))) ff.tree['bar'] = ff.make_reference([]) buff = io.BytesIO() ff.write_to(buff) buff.seek(0) content = asdf.AsdfFile().read(buff, _get_yaml_content=True) assert b"{$ref: ''}" in content
def get_write_fd(): f = generic_io.get_file(path, mode='w') assert isinstance(f, generic_io.RealFile) assert f._uri == urljoin('file:', pathname2url(path)) return f
def get_read_fd(): f = generic_io.get_file(io.open(path, 'r+b'), mode='rw') assert isinstance(f, generic_io.RealFile) assert f._uri == urljoin('file:', pathname2url(path)) return f
def get_write_fd(): f = generic_io.get_file(path, mode='w') assert isinstance(f, generic_io.RealFile) assert f._uri == urljoin('file:', pathname2url(path)) return f
def get_read_fd(): f = generic_io.get_file(io.open(path, 'r+b'), mode='rw') assert isinstance(f, generic_io.RealFile) assert f._uri == urljoin('file:', pathname2url(path)) return f
def url_mapping(self): return [('http://nowhere.org/schemas/custom/1.0.0/', urljoin('file:', pathname2url(os.path.join( TEST_DATA_PATH))) + '/{url_suffix}.yaml')]
def url_mapping(self): return [ ('http://nowhere.org/schemas/custom/1.0.0/', urljoin('file:', pathname2url(os.path.join(TEST_DATA_PATH))) + '/{url_suffix}.yaml') ]
def test_external_reference(tmpdir): exttree = { 'cool_stuff': { 'a': np.array([0, 1, 2], np.float), 'b': np.array([3, 4, 5], np.float) }, 'list_of_stuff': [ 'foobar', 42, np.array([7, 8, 9], np.float) ] } external_path = os.path.join(str(tmpdir), 'external.asdf') with asdf.AsdfFile(exttree) as ext: ext.write_to(external_path) external_path = os.path.join(str(tmpdir), 'external2.asdf') with asdf.AsdfFile(exttree) as ff: ff.write_to(external_path) tree = { # The special name "data" here must be an array. This is # included so that such validation can be ignored when we just # have a "$ref". 'data': { '$ref': 'external.asdf#/cool_stuff/a' }, 'science_data': { '$ref': 'external.asdf#/cool_stuff/a' }, 'science_data2': { '$ref': 'external2.asdf#/cool_stuff/a' }, 'foobar': { '$ref': 'external.asdf#/list_of_stuff/0', }, 'answer': { '$ref': 'external.asdf#/list_of_stuff/1' }, 'array': { '$ref': 'external.asdf#/list_of_stuff/2', }, 'whole_thing': { '$ref': 'external.asdf#' }, 'myself': { '$ref': '#', }, 'internal': { '$ref': '#science_data' } } def do_asserts(ff): assert 'unloaded' in repr(ff.tree['science_data']) assert 'unloaded' in str(ff.tree['science_data']) assert len(ff._external_asdf_by_uri) == 0 assert_array_equal(ff.tree['science_data'], exttree['cool_stuff']['a']) assert len(ff._external_asdf_by_uri) == 1 with pytest.raises((ValueError, RuntimeError)): # Assignment destination is readonly ff.tree['science_data'][0] = 42 assert_array_equal(ff.tree['science_data2'], exttree['cool_stuff']['a']) assert len(ff._external_asdf_by_uri) == 2 assert ff.tree['foobar']() == 'foobar' assert ff.tree['answer']() == 42 assert_array_equal(ff.tree['array'], exttree['list_of_stuff'][2]) assert_tree_match(ff.tree['whole_thing'](), exttree) assert_array_equal( ff.tree['whole_thing']['cool_stuff']['a'], exttree['cool_stuff']['a']) assert_array_equal( ff.tree['myself']['science_data'], exttree['cool_stuff']['a']) # Make sure that referencing oneself doesn't make another call # to disk. assert len(ff._external_asdf_by_uri) == 2 assert_array_equal(ff.tree['internal'], exttree['cool_stuff']['a']) with asdf.AsdfFile(tree, uri=pathname2url( os.path.join(str(tmpdir), 'main.asdf'))) as ff: do_asserts(ff) internal_path = os.path.join(str(tmpdir), 'main.asdf') ff.write_to(internal_path) with asdf.AsdfFile.read(internal_path) as ff: do_asserts(ff) with asdf.AsdfFile.read(internal_path) as ff: assert len(ff._external_asdf_by_uri) == 0 ff.resolve_references() assert len(ff._external_asdf_by_uri) == 2 assert isinstance(ff.tree['data'], ndarray.NDArrayType) assert isinstance(ff.tree['science_data'], ndarray.NDArrayType) assert_array_equal(ff.tree['science_data'], exttree['cool_stuff']['a']) assert_array_equal(ff.tree['science_data2'], exttree['cool_stuff']['a']) assert ff.tree['foobar'] == 'foobar' assert ff.tree['answer'] == 42 assert_array_equal(ff.tree['array'], exttree['list_of_stuff'][2]) assert_tree_match(ff.tree['whole_thing'], exttree) assert_array_equal( ff.tree['whole_thing']['cool_stuff']['a'], exttree['cool_stuff']['a']) assert_array_equal( ff.tree['myself']['science_data'], exttree['cool_stuff']['a']) assert_array_equal(ff.tree['internal'], exttree['cool_stuff']['a'])
return None return _map_func func = _make_map_func(mapping) else: raise ValueError("Invalid mapping '{0}'".format(mapping)) normalized.append(func) return tuple(normalized) def __call__(self, input): for mapper in self._mapping: output = mapper(input) if output is not None: return output return input def __hash__(self): return hash(self._mapping) DEFAULT_URL_MAPPING = [ (constants.STSCI_SCHEMA_URI_BASE, urljoin('file:', pathname2url(os.path.join( SCHEMA_PATH, 'stsci.edu'))) + '/{url_suffix}.yaml') ] default_url_mapping = Resolver(DEFAULT_URL_MAPPING, 'url')
return None return _map_func func = _make_map_func(mapping) else: raise ValueError("Invalid mapping '{0}'".format(mapping)) normalized.append(func) return tuple(normalized) def __call__(self, input): for mapper in self._mapping: output = mapper(input) if output is not None: return output return input def __hash__(self): return hash(self._mapping) DEFAULT_URL_MAPPING = [ (constants.STSCI_SCHEMA_URI_BASE, urljoin('file:', pathname2url(os.path.join(SCHEMA_PATH, 'stsci.edu'))) + '/{url_suffix}.yaml') ] default_url_mapping = Resolver(DEFAULT_URL_MAPPING, 'url')