def test_array_inline_threshold():
with asdf.config_context() as config:
assert config.array_inline_threshold == asdf.config.DEFAULT_ARRAY_INLINE_THRESHOLD
config.array_inline_threshold = 10
assert get_config().array_inline_threshold == 10
config.array_inline_threshold = None
assert get_config().array_inline_threshold is None
def test_core_schema(tmp_path):
# Set temporary asdf file
file_path = tmp_path / "test.asdf"
wfi_image = utils.mk_level2_image(arrays=(10, 10))
with asdf.AsdfFile() as af:
af.tree = {'roman': wfi_image}
with pytest.raises(ValidationError):
af.tree['roman'].meta.telescope = 'NOTROMAN'
af.tree['roman'].meta['telescope'] = 'NOTROMAN'
with pytest.raises(ValidationError):
af.write_to(file_path)
af.tree['roman'].meta.telescope = 'ROMAN'
af.write_to(file_path)
# Now mangle the file
with open(file_path, 'rb') as fp:
fcontents = fp.read()
romanloc = fcontents.find(bytes('ROMAN', 'utf-8'))
newcontents = fcontents[:romanloc] + \
bytes('X', 'utf-8') + fcontents[romanloc + 1:]
with open(file_path, 'wb') as fp:
fp.write(newcontents)
with pytest.raises(ValidationError):
with datamodels.open(file_path) as model:
pass
asdf.get_config().validate_on_read = False
with datamodels.open(file_path) as model:
assert model.meta.telescope == 'XOMAN'
asdf.get_config().validate_on_read = True
def test_validate_on_read():
with asdf.config_context() as config:
assert config.validate_on_read == asdf.config.DEFAULT_VALIDATE_ON_READ
config.validate_on_read = False
assert get_config().validate_on_read is False
config.validate_on_read = True
assert get_config().validate_on_read is True
def test_legacy_fill_schema_defaults():
with asdf.config_context() as config:
assert config.legacy_fill_schema_defaults == asdf.config.DEFAULT_LEGACY_FILL_SCHEMA_DEFAULTS
config.legacy_fill_schema_defaults = False
assert get_config().legacy_fill_schema_defaults is False
config.legacy_fill_schema_defaults = True
assert get_config().legacy_fill_schema_defaults is True
def test_validate_on_read(tmpdir):
tmpfile = str(tmpdir.join('invalid.fits'))
content = """
invalid_software: !core/software-1.0.0
name: Minesweeper
version: 3
"""
buff = yaml_to_asdf(content)
hdul = fits.HDUList()
data = np.array(buff.getbuffer(), dtype=np.uint8)[None, :]
fmt = '{}B'.format(len(data[0]))
column = fits.Column(array=data, format=fmt, name='ASDF_METADATA')
hdu = fits.BinTableHDU.from_columns([column], name='ASDF')
hdul.append(hdu)
hdul.writeto(tmpfile)
for open_method in [asdf.open, fits_embed.AsdfInFits.open]:
with pytest.raises(ValidationError):
get_config().validate_on_read = True
with open_method(tmpfile):
pass
get_config().validate_on_read = False
with open_method(tmpfile) as af:
assert af["invalid_software"]["name"] == "Minesweeper"
assert af["invalid_software"]["version"] == 3
def test_config_context():
assert get_config().validate_on_read is True
with asdf.config_context() as config:
config.validate_on_read = False
assert get_config().validate_on_read is False
assert get_config().validate_on_read is True
def test_global_config():
assert get_config().validate_on_read is True
get_config().validate_on_read = False
assert get_config().validate_on_read is False
with asdf.config_context() as config:
assert config.validate_on_read is False
config.validate_on_read = True
assert get_config().validate_on_read is True
assert get_config().validate_on_read is False
def test_config_context_nested():
assert get_config().validate_on_read is True
with asdf.config_context() as config1:
config1.validate_on_read = False
with asdf.config_context() as config2:
config2.validate_on_read = True
with asdf.config_context() as config3:
config3.validate_on_read = False
assert get_config().validate_on_read is False
assert get_config().validate_on_read is True
def __init__(self, standard=None):
if standard is None:
standard = "a"
self._standard = standard
mappings = {}
for cls in self.types:
key = "http://QualityOrganization/schemas/default/" + cls.name + "-1.0.0.yaml"
schema_path = "custom_module/schemas/standards/" + self._standard + "/" + cls.name + "-1.0.0.yaml"
schema = generic_io.get_file(schema_path).read().decode('utf-8')
mappings[key] = schema
asdf.get_config().add_resource_mapping(mappings)
def test_load_schema_from_resource_mapping():
content = """
id: http://somewhere.org/schemas/razmataz-1.0.0
type: object
properties:
foo:
type: string
bar:
type: boolean
""".encode("utf-8")
get_config().add_resource_mapping(
{"http://somewhere.org/schemas/razmataz-1.0.0": content})
s = schema.load_schema("http://somewhere.org/schemas/razmataz-1.0.0")
assert s["id"] == "http://somewhere.org/schemas/razmataz-1.0.0"
def test_config_context_threaded():
assert get_config().validate_on_read is True
thread_value = None
def worker():
nonlocal thread_value
thread_value = get_config().validate_on_read
with asdf.config_context() as config:
config.validate_on_read = False
pass
with asdf.config_context() as config:
config.validate_on_read = False
thread = threading.Thread(target=worker)
thread.start()
thread.join()
assert thread_value is True
assert get_config().validate_on_read is True
def test_resources():
resources_root = Path(__file__).parent.parent / "resources"
resource_manager = asdf.get_config().resource_manager
for resource_path in resources_root.glob("**/*.yaml"):
with resource_path.open("rb") as f:
resource_content = f.read()
resource = yaml.safe_load(resource_content)
resource_uri = resource["id"]
assert resource_manager[resource_uri] == resource_content
def test_open_validate_on_read(tmpdir):
content = """
invalid_software: !core/software-1.0.0
name: Minesweeper
version: 3
"""
buff = yaml_to_asdf(content)
with pytest.raises(ValidationError):
get_config().validate_on_read = True
with asdf.open(buff):
pass
buff.seek(0)
get_config().validate_on_read = False
with asdf.open(buff) as af:
assert af["invalid_software"]["name"] == "Minesweeper"
assert af["invalid_software"]["version"] == 3
def extract_2d_spectrum(data, ll_x, ll_y, ur_x, ur_y, ll_l = 1.1,
ur_l = 1.7, order = 1, specwcs_ref = "wfc3_ir_specwcs.asdf"):
"""
Function to do a simple box cutout around a 2D spectrum.
The input lower and upper x and y bounds are pixel coordinates from the
direct image. The upper and lower wavelength bounds are in microns.
TODO: Change this to take grism as input instead of lower and upper wavelengths
and specwcs reference file, with default wavelength range based on grism.
Currently assumes the G141 reference file and wavelength ranged that I've been
working with.
"""
asdf.get_config().add_extension(DISPXY_Extension())
specwcs = asdf.open(specwcs_ref).tree
displ = specwcs['displ']
dispx = specwcs['dispx']
dispy = specwcs['dispy']
invdispl = specwcs['invdispl']
invdispx = specwcs['invdispx']
invdispy = specwcs['invdispy']
orders = specwcs['order']
det2det = WFC3IRForwardGrismDispersion(orders,
lmodels=displ,
xmodels=invdispx,
ymodels=dispy)
det2det.inverse = WFC3IRBackwardGrismDispersion(orders,
lmodels=invdispl,
xmodels=dispx,
ymodels=dispy)
ll = det2det.inverse.evaluate(ll_x, ll_y, ll_l, order)
ur = det2det.inverse.evaluate(ur_x, ur_y, ur_l, order)
print(ll, ur)
return data[int(ll[1]):int(ur[1])+1, int(ll[0]):int(ur[0])+1]
def test_asdf_file_extensions():
af = AsdfFile()
assert af.extensions == get_config().extensions
extension = TestExtension(
extension_uri="asdf://somewhere.org/extensions/foo-1.0")
for arg in ([extension], extension, AsdfExtensionList([extension])):
af = AsdfFile(extensions=arg)
assert af.extensions[0] == ExtensionProxy(extension)
assert af.extensions[1:] == get_config().extensions
af = AsdfFile()
af.extensions = arg
assert af.extensions[0] == ExtensionProxy(extension)
assert af.extensions[1:] == get_config().extensions
# This use case is a little silly, but in the future it will be
# possible to disable extensions globally and passing the URI
# will enable them on an individual file basis. Currently all
# we can do with the URI is put the extension at the head of
# list so that it has priority.
with config_context() as config:
config.add_extension(extension)
for arg in ([extension.extension_uri], extension.extension_uri):
af = AsdfFile(extensions=arg)
assert af.extensions[0] == ExtensionProxy(extension)
af = AsdfFile()
af.extensions = arg
assert af.extensions[0] == ExtensionProxy(extension)
for arg in (object(), [object()]):
with pytest.raises(TypeError):
AsdfFile(extensions=arg)
with pytest.raises(KeyError):
AsdfFile(extensions="not-a-URI")
def test_manifests():
manifests_root = (Path(__file__).parent.parent / "resources" / "manifests")
resource_manager = asdf.get_config().resource_manager
for manifest_path in manifests_root.glob("*.yaml"):
with manifest_path.open("rb") as f:
manifest_content = f.read()
manifest = yaml.safe_load(manifest_content)
manifest_schema = asdf.schema.load_schema(
"asdf://asdf-format.org/core/schemas/extension_manifest-1.0")
# The manifest must be valid against its own schema:
asdf.schema.validate(manifest, schema=manifest_schema)
for tag_definition in manifest["tags"]:
# The tag's schema must be available:
assert tag_definition["schema_uri"] in resource_manager
def test_extensions():
package_and_uri_pairs = {(e.package_name, e.extension_uri)
for e in asdf.get_config().extensions}
assert ("asdf-astropy", "http://astropy.org/asdf/extensions/astropy-1.0"
) in package_and_uri_pairs
assert ("asdf-astropy", "http://stsci.edu/asdf/extensions/transform-1.0.0"
) in package_and_uri_pairs
assert ("asdf-astropy", "http://stsci.edu/asdf/extensions/transform-1.1.0"
) in package_and_uri_pairs
assert ("asdf-astropy", "http://stsci.edu/asdf/extensions/transform-1.2.0"
) in package_and_uri_pairs
assert ("asdf-astropy", "http://stsci.edu/asdf/extensions/transform-1.3.0"
) in package_and_uri_pairs
assert ("asdf-astropy", "http://stsci.edu/asdf/extensions/transform-1.4.0"
) in package_and_uri_pairs
assert ("asdf-astropy", "http://stsci.edu/asdf/extensions/transform-1.5.0"
) in package_and_uri_pairs
def test_manifest_valid(manifest):
schema = asdf.schema.load_schema(
"asdf://asdf-format.org/core/schemas/extension_manifest-1.0.0")
asdf.schema.validate(manifest, schema=schema)
assert "title" in manifest
assert "description" in manifest
for tag in manifest["tags"]:
# Check that the schema exists:
assert tag["schema_uri"] in asdf.get_config().resource_manager
# These are not required by the manifest schema but we're holding ourselves
# to a higher standard:
assert "title" in tag
assert "description" in tag
assert tag["tag_uri"].startswith(
"asdf://stsci.edu/datamodels/roman/tags/")
def _http_to_temp(init, mode, uri=None):
"""
Stream the content of an http or https URL to a temporary file.
Parameters
----------
init : str
HTTP or HTTPS URL.
mode : str
ASDF file mode. The temporary file will always be opened
in w+b mode, but the resulting GenericFile will report itself
writable based on this value.
uri : str, optional
URI against which relative paths within the file are
resolved. If None, the init value will be used.
Returns
-------
RealFile
Temporary file.
"""
from asdf import get_config
fd = tempfile.NamedTemporaryFile("w+b")
block_size = get_config().io_block_size
if block_size == -1:
try:
block_size = os.fstat(fd.fileno()).st_blksize
except Exception:
block_size = io.DEFAULT_BUFFER_SIZE
try:
# This method is only called with http and https schemes:
with urlopen(init) as response: # nosec
chunk = response.read(block_size)
while len(chunk) > 0:
fd.write(chunk)
chunk = response.read(block_size)
fd.seek(0)
return RealFile(fd, mode, close=True, uri=uri or init)
except Exception:
fd.close()
raise
def test_serialization_context():
extension_manager = ExtensionManager([])
context = SerializationContext("1.4.0", extension_manager)
assert context.version == "1.4.0"
assert context.extension_manager is extension_manager
assert context._extensions_used == set()
extension = get_config().extensions[0]
context._mark_extension_used(extension)
assert context._extensions_used == {extension}
context._mark_extension_used(extension)
assert context._extensions_used == {extension}
context._mark_extension_used(extension.delegate)
assert context._extensions_used == {extension}
with pytest.raises(TypeError):
context._mark_extension_used(object())
with pytest.raises(ValueError):
SerializationContext("0.5.4", extension_manager)
def enable_quality_standard(name: str,
version: Union[AsdfVersion, str] = None):
"""Enable a quality standard.
All corresponding schemas will be used for validation during serialization and
deserialization of a weldx file.
Parameters
----------
name :
Name of the quality standard
version :
Requested standard version. If `None` is provided, the latest will be used.
"""
standard = Config._standards[name]
manifest_mapping, schema_mapping = standard.get_mappings(version)
asdf_config = asdf.get_config()
asdf_config.add_resource_mapping(manifest_mapping)
asdf_config.add_resource_mapping(schema_mapping)
def __init__(self, fd, mode, close=False, uri=None):
"""
Parameters
----------
fd : file-like object
The particular kind of file-like object must match the
subclass of `GenericFile` being instantiated.
mode : str
Must be ``"r"`` (read), ``"w"`` (write), or ``"rw"``
(read/write).
close : bool, optional
When ``True``, close the given `fd` in the ``__exit__``
method, i.e. at the end of the with block. Should be set
to ``True`` when this object "owns" the file object.
Default: ``False``.
uri : str, optional
The file path or URI used to open the file. This is used
to resolve relative URIs when the file refers to external
sources.
"""
if not _check_bytes(fd, mode):
raise ValueError(
"File-like object must be opened in binary mode.")
# can't import at the top level due to circular import
from .config import get_config
self._asdf_get_config = get_config
self._fd = fd
self._mode = mode
self._close = close
self._size = None
self._uri = uri
self.block_size = get_config().io_block_size
def create_grism_specwcs(conffile="",
pupil=None,
direct_filter=None,
author="STScI",
history="",
outname=None):
"""
Note: This code is shamelessly stolen from the jwreftools package
(see https://github.com/spacetelescope/jwreftools/) and adapted for use
on HST GRISMCONF files. The docstrings and comments have not yet been
updated accordingly.
Create an asdf reference file to hold grism configuration information. No
sensitivity information is included
Note: The orders are named alphabetically, i.e. Order A, Order B
There are also sensativity fits files which are tables of wavelength,
sensativity, and error. These are specified in the conffile but will
not be read in and saved in the output reference file for now.
It's possible they may be included in the future, either here or as
a separate reference files. Their use here would be to help define the
min and max wavelengths which set the extent of the dispersed trace on
the grism image. Convolving the sensitiviy file with the filter throughput
allows one to calculate the wavelength of minimum throughput which defines
the edges of the trace.
direct_filter is not specified because it assumes that the wedge
information (wx,wy) is included in the conf file in one of the key-value
pairs, where the key includes the beam designation
this reference file also contains the polynomial model which is
appropriate for the coefficients which are listed.
wavelength = DISPL(order,x0,y0,t)
dx = DISPX(order,x0,y0,t)
dy = DISPY(order,x0,y0,t)
t = INVDISPX(order,x0,y0,dx)
t = INVDISPY(order,x0,y0,dy)
t = INVDISL(order,x0,y0, wavelength)
Parameters
----------
conffile : str
The text file with configuration information, formatted as aXe expects
pupil : str
Name of the grism the conffile corresponds to
Taken from the conffile name if not specified
module : str
Name of the Nircam module
Taken from the conffile name if not specified
author : str
The name of the author
history : str
A comment about the refrence file to be saved with the meta information
outname : str
Output name for the reference file
Returns
-------
fasdf : asdf.AsdfFile(WFC3IRGrismModel)
"""
if outname is None:
outname = "wfc3_ir_specwcs.asdf"
if not history:
history = "Created from {0:s}".format(conffile)
# if pupil is none get from filename like NIRCAM_modB_R.conf
if pupil is None:
pupil = "GRISM" + conffile.split(".")[0][-1]
print("Pupil is {}".format(pupil))
ref_kw = common_reference_file_keywords(
reftype="specwcs",
title="HST IR Grism Parameters",
description="{0:s} dispersion models".format(pupil),
exp_type="WFC3_IR",
author=author,
model_type="WFC3IRGrismModel",
fname=direct_filter,
pupil=pupil,
filename=outname,
)
# get all the key-value pairs from the input file
conf = dict_from_file(conffile)
beamdict = split_order_info(conf)
# Get x and y offsets from the filter, if necessary
if direct_filter is not None:
wx = beamdict["WEDGE"][direct_filter][0]
wy = beamdict["WEDGE"][direct_filter][0]
else:
wx, wy = 0, 0
beamdict.pop("WEDGE")
# beam = re.compile('^(?:[+\-]){0,1}[a-zA-Z0-9]{0,1}$') # match beam only
# read in the sensitivity tables to save their content
# they currently have names like this: NIRCam.A.1st.sensitivity.fits
# translated as inst.beam/order.param
temp = dict()
etoken = re.compile("^[a-zA-Z]*_(?:[+\-]){1,1}[1,2]{1,1}") # find beam key
for b, bdict in beamdict.items():
temp[b] = dict()
# add the new beam information to beamdict and remove spurious beam info
for k in temp:
for kk in temp[k]:
if etoken.match(kk):
kk = kk.replace("_{}".format(k), "")
beamdict[k][kk] = temp[k][kk]
# for NIRCAM, the R and C grism coefficients contain zeros where
# the dispersion is in the opposite direction. Meaning, the GRISMR,
# which disperses along ROWS has coefficients of zero in the y models
# and vice versa.
#
# There are separate reference files for each grism. Depending on the grism
# dispersion direction you either want to use the dx from source center or
# the dy from source center in the inverse dispersion relationship which is
# used to calculate the t value needed to calculate the wavelength at that
# pixel.
# The model creation here takes all of this into account by looking at the
# GRISM[R/C] the file is used for and creating a reference model with the
# appropriate dispersion direction in use. This eliminates having to decide
# which direction to calculatethe dispersion from given the input x,y
# pixel in the dispersed image.
orders = beamdict.keys()
# dispersion models valid per order and direction saved to reference file
# Forward
invdispl = []
invdispx = []
invdispy = []
# Backward
displ = []
dispx = []
dispy = []
for order in orders:
# convert the displ wavelengths to microns if the input
# file is still in angstroms
l0 = beamdict[order]['DISPL'][0] / 10000.
l1 = beamdict[order]['DISPL'][1] / 10000.
# create polynomials using the coefficients of each order
# This holds the wavelength lookup coeffs
# This model is INVDISPL for backward and returns t
# This model should be DISPL for forward and returns wavelength
if l1 == 0:
lmodel = Polynomial1D(1, c0=0, c1=0)
else:
lmodel = Polynomial1D(1, c0=-l0 / l1, c1=1. / l1)
invdispl.append(lmodel)
lmodel = Polynomial1D(1, c0=l0, c1=l1)
displ.append(lmodel)
# This holds the x coefficients, for the R grism this model is the
# the INVDISPX returning t, for the C grism this model is the DISPX
e = beamdict[order]['DISPX']
xmodel = DISPXY_Model(e, wx)
dispx.append(xmodel)
inv_xmodel = DISPXY_Model(e, wx, inv=True)
invdispx.append(inv_xmodel)
# This holds the y coefficients, for the C grism, this model is
# the INVDISPY, returning t, for the R grism, this model is the DISPY
e = beamdict[order]['DISPY']
ymodel = DISPXY_Model(e, wy)
dispy.append(ymodel)
inv_ymodel = DISPXY_Model(e, wy, inv=True)
invdispy.append(ymodel)
# change the orders into translatable integers
# so that we can look up the order with the proper index
oo = [int(o) for o in beamdict]
# We need to register the converter for the DISPXY_Model class with asdf
asdf.get_config().add_extension(DISPXY_Extension())
ref = WFC3IRGrismModel()
ref.meta.update(ref_kw)
# This reference file is good for NRC_WFSS and TSGRISM modes
ref.meta.exposure.p_exptype = "NRC_WFSS|NRC_TSGRISM"
ref.meta.input_units = u.micron
ref.meta.output_units = u.micron
ref.displ = displ
ref.dispx = dispx
print(ref.dispx)
ref.dispy = dispy
print(ref.dispy)
ref.invdispx = invdispx
ref.invdispy = invdispy
ref.invdispl = invdispl
ref.order = oo
history = HistoryEntry({
'description': history,
'time': datetime.datetime.utcnow()
})
software = Software({
'name': 'nircam_reftools.py',
'author': author,
'homepage': 'https://github.com/spacetelescope/jwreftools',
'version': '0.7.1'
})
history['software'] = software
ref.history = [history]
ref.to_asdf(outname)
ref.validate()
def test_schema_integration(schema_path):
content = schema_path.read_bytes()
schema = yaml.safe_load(content)
asdf_content = asdf.get_config().resource_manager[schema["id"]]
assert asdf_content == content
def manifest():
return yaml.safe_load(asdf.get_config().resource_manager[
"asdf://stsci.edu/datamodels/roman/manifests/datamodels-1.0"])
def test_manifest_integration(manifest_path):
content = manifest_path.read_bytes()
manifest = yaml.safe_load(content)
asdf_content = asdf.get_config().resource_manager[manifest["id"]]
assert asdf_content == content
"""
Test features of the schemas not covered by the metaschema.
"""
from collections.abc import Mapping
import asdf
import pytest
import yaml
SCHEMA_URI_PREFIX = "asdf://stsci.edu/datamodels/roman/schemas/"
METASCHEMA_URI = "asdf://stsci.edu/datamodels/roman/schemas/rad_schema-1.0.0"
SCHEMA_URIS = [
u for u in asdf.get_config().resource_manager
if u.startswith(SCHEMA_URI_PREFIX) and u != METASCHEMA_URI
]
@pytest.fixture(scope="session", params=SCHEMA_URIS)
def schema_content(request):
return asdf.get_config().resource_manager[request.param]
@pytest.fixture(scope="session", params=SCHEMA_URIS)
def schema(request):
return yaml.safe_load(asdf.get_config().resource_manager[request.param])
@pytest.fixture(scope="session")
def valid_tag_uris(manifest):
uris = {t["tag_uri"] for t in manifest["tags"]}
uris.update([
from custom_module.custom_types import CustomClass
my_float = 12.45
my_list = [3, 4, 6]
my_dict = {"a": 1, "b": 42}
cc = CustomClass(2.45, [2, 8, 64])
tree = {
"cc": cc,
}
s = load_schema("custom_module/schemas/CustomClass-alternative.yaml")
s_string = generic_io.get_file(
"custom_module/schemas/CustomClass-alternative.yaml").read().decode(
'utf-8')
asdf.get_config().add_resource_mapping({
"http://CustomOrganization/schemas/CustomStandard/CustomClass-1.0.0":
s_string
})
print("Script --- Create AsdfFile instance")
af = asdf.AsdfFile(
tree,
extensions=CustomExtensions(),
custom_schema=f"custom_module\\schemas\\CustomSchema.yaml",
)
print("\nScript --- Call write_to of AsdfFile instance")
af.write_to("test.yml")
def schema(request):
return yaml.safe_load(asdf.get_config().resource_manager[request.param])
def schema_content(request):
return asdf.get_config().resource_manager[request.param]
