def test_external_reference_invalid_fragment(tmpdir):
exttree = {
'list_of_stuff': [
'foobar',
42,
np.array([7, 8, 9], np.float)
]
}
external_path = os.path.join(str(tmpdir), 'external.asdf')
with asdf.AsdfFile(exttree) as ff:
ff.write_to(external_path)
tree = {
'foo': {
'$ref': 'external.asdf#/list_of_stuff/a'
}
}
with asdf.AsdfFile(tree, uri=pathname2url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
with pytest.raises(ValueError):
ff.resolve_references()
tree = {
'foo': {
'$ref': 'external.asdf#/list_of_stuff/3'
}
}
with asdf.AsdfFile(tree, uri=pathname2url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
with pytest.raises(ValueError):
ff.resolve_references()
def get_read_fd():
f = generic_io.get_file(path, mode='r')
assert isinstance(f, generic_io.RealFile)
assert f._uri == urljoin('file:', pathname2url(path))
# This is to check for a "feature" in Python 3.x that reading zero
# bytes from a socket causes it to stop. We have code in generic_io.py
# to workaround it.
f.read(0)
return f
def get_read_fd():
f = generic_io.get_file(path, mode='r')
assert isinstance(f, generic_io.RealFile)
assert f._uri == urljoin('file:', pathname2url(path))
# This is to check for a "feature" in Python 3.x that reading zero
# bytes from a socket causes it to stop. We have code in generic_io.py
# to workaround it.
f.read(0)
return f
def __init__(self, fd, mode, close=True, uri=None):
super(RealFile, self).__init__(fd, mode, close=close, uri=uri)
stat = os.fstat(fd.fileno())
if sys.platform.startswith('win'):
# There appears to be reliable way to get block size on Windows,
# so just choose a reasonable default
self._blksize = io.DEFAULT_BUFFER_SIZE
else:
self._blksize = stat.st_blksize
self._size = stat.st_size
if (uri is None and
isinstance(fd.name, six.string_types) and
os.path.exists(fd.name)):
self._uri = urlparse.urljoin(
'file:', pathname2url(os.path.abspath(fd.name)))
def test_external_reference_invalid(tmpdir):
tree = {
'foo': {
'$ref': 'fail.asdf'
}
}
ff = asdf.AsdfFile(tree)
with pytest.raises(ValueError):
ff.resolve_references()
ff = asdf.AsdfFile(tree, uri="http://nowhere.com/")
with pytest.raises(IOError):
ff.resolve_references()
ff = asdf.AsdfFile(tree, uri=pathname2url(
os.path.join(str(tmpdir), 'main.asdf')))
with pytest.raises(IOError):
ff.resolve_references()
def test_internal_reference():
tree = {
'foo': 2,
'bar': {'$ref': '#'}
}
ff = asdf.AsdfFile(tree)
ff.find_references()
assert isinstance(ff.tree['bar'], reference.Reference)
ff.resolve_references()
assert ff.tree['bar']['foo'] == 2
tree = {
'foo': 2
}
ff = asdf.AsdfFile(tree, uri=pathname2url(os.path.abspath("test.asdf")))
ff.tree['bar'] = ff.make_reference([])
buff = io.BytesIO()
ff.write_to(buff)
buff.seek(0)
content = asdf.AsdfFile().read(buff, _get_yaml_content=True)
assert b"{$ref: ''}" in content
def get_write_fd():
f = generic_io.get_file(path, mode='w')
assert isinstance(f, generic_io.RealFile)
assert f._uri == urljoin('file:', pathname2url(path))
return f
def get_read_fd():
f = generic_io.get_file(io.open(path, 'r+b'), mode='rw')
assert isinstance(f, generic_io.RealFile)
assert f._uri == urljoin('file:', pathname2url(path))
return f
def get_write_fd():
f = generic_io.get_file(path, mode='w')
assert isinstance(f, generic_io.RealFile)
assert f._uri == urljoin('file:', pathname2url(path))
return f
def get_read_fd():
f = generic_io.get_file(io.open(path, 'r+b'), mode='rw')
assert isinstance(f, generic_io.RealFile)
assert f._uri == urljoin('file:', pathname2url(path))
return f
def url_mapping(self):
return [('http://nowhere.org/schemas/custom/1.0.0/',
urljoin('file:', pathname2url(os.path.join(
TEST_DATA_PATH))) + '/{url_suffix}.yaml')]
def url_mapping(self):
return [
('http://nowhere.org/schemas/custom/1.0.0/',
urljoin('file:', pathname2url(os.path.join(TEST_DATA_PATH))) +
'/{url_suffix}.yaml')
]
def test_external_reference(tmpdir):
exttree = {
'cool_stuff': {
'a': np.array([0, 1, 2], np.float),
'b': np.array([3, 4, 5], np.float)
},
'list_of_stuff': [
'foobar',
42,
np.array([7, 8, 9], np.float)
]
}
external_path = os.path.join(str(tmpdir), 'external.asdf')
with asdf.AsdfFile(exttree) as ext:
ext.write_to(external_path)
external_path = os.path.join(str(tmpdir), 'external2.asdf')
with asdf.AsdfFile(exttree) as ff:
ff.write_to(external_path)
tree = {
# The special name "data" here must be an array. This is
# included so that such validation can be ignored when we just
# have a "$ref".
'data': {
'$ref': 'external.asdf#/cool_stuff/a'
},
'science_data': {
'$ref': 'external.asdf#/cool_stuff/a'
},
'science_data2': {
'$ref': 'external2.asdf#/cool_stuff/a'
},
'foobar': {
'$ref': 'external.asdf#/list_of_stuff/0',
},
'answer': {
'$ref': 'external.asdf#/list_of_stuff/1'
},
'array': {
'$ref': 'external.asdf#/list_of_stuff/2',
},
'whole_thing': {
'$ref': 'external.asdf#'
},
'myself': {
'$ref': '#',
},
'internal': {
'$ref': '#science_data'
}
}
def do_asserts(ff):
assert 'unloaded' in repr(ff.tree['science_data'])
assert 'unloaded' in str(ff.tree['science_data'])
assert len(ff._external_asdf_by_uri) == 0
assert_array_equal(ff.tree['science_data'], exttree['cool_stuff']['a'])
assert len(ff._external_asdf_by_uri) == 1
with pytest.raises((ValueError, RuntimeError)):
# Assignment destination is readonly
ff.tree['science_data'][0] = 42
assert_array_equal(ff.tree['science_data2'], exttree['cool_stuff']['a'])
assert len(ff._external_asdf_by_uri) == 2
assert ff.tree['foobar']() == 'foobar'
assert ff.tree['answer']() == 42
assert_array_equal(ff.tree['array'], exttree['list_of_stuff'][2])
assert_tree_match(ff.tree['whole_thing'](), exttree)
assert_array_equal(
ff.tree['whole_thing']['cool_stuff']['a'],
exttree['cool_stuff']['a'])
assert_array_equal(
ff.tree['myself']['science_data'],
exttree['cool_stuff']['a'])
# Make sure that referencing oneself doesn't make another call
# to disk.
assert len(ff._external_asdf_by_uri) == 2
assert_array_equal(ff.tree['internal'], exttree['cool_stuff']['a'])
with asdf.AsdfFile(tree, uri=pathname2url(
os.path.join(str(tmpdir), 'main.asdf'))) as ff:
do_asserts(ff)
internal_path = os.path.join(str(tmpdir), 'main.asdf')
ff.write_to(internal_path)
with asdf.AsdfFile.read(internal_path) as ff:
do_asserts(ff)
with asdf.AsdfFile.read(internal_path) as ff:
assert len(ff._external_asdf_by_uri) == 0
ff.resolve_references()
assert len(ff._external_asdf_by_uri) == 2
assert isinstance(ff.tree['data'], ndarray.NDArrayType)
assert isinstance(ff.tree['science_data'], ndarray.NDArrayType)
assert_array_equal(ff.tree['science_data'], exttree['cool_stuff']['a'])
assert_array_equal(ff.tree['science_data2'], exttree['cool_stuff']['a'])
assert ff.tree['foobar'] == 'foobar'
assert ff.tree['answer'] == 42
assert_array_equal(ff.tree['array'], exttree['list_of_stuff'][2])
assert_tree_match(ff.tree['whole_thing'], exttree)
assert_array_equal(
ff.tree['whole_thing']['cool_stuff']['a'],
exttree['cool_stuff']['a'])
assert_array_equal(
ff.tree['myself']['science_data'],
exttree['cool_stuff']['a'])
assert_array_equal(ff.tree['internal'], exttree['cool_stuff']['a'])
return None
return _map_func
func = _make_map_func(mapping)
else:
raise ValueError("Invalid mapping '{0}'".format(mapping))
normalized.append(func)
return tuple(normalized)
def __call__(self, input):
for mapper in self._mapping:
output = mapper(input)
if output is not None:
return output
return input
def __hash__(self):
return hash(self._mapping)
DEFAULT_URL_MAPPING = [
(constants.STSCI_SCHEMA_URI_BASE,
urljoin('file:', pathname2url(os.path.join(
SCHEMA_PATH, 'stsci.edu'))) + '/{url_suffix}.yaml')
]
default_url_mapping = Resolver(DEFAULT_URL_MAPPING, 'url')
return None
return _map_func
func = _make_map_func(mapping)
else:
raise ValueError("Invalid mapping '{0}'".format(mapping))
normalized.append(func)
return tuple(normalized)
def __call__(self, input):
for mapper in self._mapping:
output = mapper(input)
if output is not None:
return output
return input
def __hash__(self):
return hash(self._mapping)
DEFAULT_URL_MAPPING = [
(constants.STSCI_SCHEMA_URI_BASE,
urljoin('file:', pathname2url(os.path.join(SCHEMA_PATH, 'stsci.edu'))) +
'/{url_suffix}.yaml')
]
default_url_mapping = Resolver(DEFAULT_URL_MAPPING, 'url')