def dataset(array, identity_gwcs):
meta = {
'inventory': {
'bucket': 'data',
'datasetId': 'test_dataset',
'primaryProposalId': 'test_proposal',
'asdfObjectKey': 'test_proposal/test_dataset/test_dataset.asdf'
},
'headers': Table()
}
identity_gwcs.array_shape = array.shape
identity_gwcs.pixel_shape = array.shape[::-1]
ds = Dataset(array, wcs=identity_gwcs, meta=meta)
# Sanity checks
assert ds.data is array
assert ds.wcs is identity_gwcs
ds._file_manager = FileManager(['test1.fits'],
0,
'float',
array.shape,
loader=AstropyFITSLoader)
return ds
def test_init_missing_meta_keys(identity_gwcs):
data = np.zeros(identity_gwcs.array_shape)
with pytest.raises(ValueError, match=".*must contain the headers table."):
Dataset(data, wcs=identity_gwcs, meta={'inventory': {}})
with pytest.raises(ValueError, match=".*must contain the inventory record."):
Dataset(data, wcs=identity_gwcs, meta={'headers': {}})
def test_array_container():
dataset = Dataset.from_directory(os.path.join(rootdir, 'EIT'))
assert dataset.array_container is dataset._array_container
with pytest.raises(AttributeError):
dataset.array_container = 10
assert len(dataset.array_container.filenames) == 11
assert len(dataset.filenames) == 11
def dataset_4d(identity_gwcs_4d, empty_meta):
shape = (50, 60, 70, 80)
x = np.ones(shape)
array = da.from_array(x, tuple(shape))
identity_gwcs_4d.pixel_shape = array.shape[::-1]
identity_gwcs_4d.array_shape = array.shape
return Dataset(array, wcs=identity_gwcs_4d, meta=empty_meta)
def dataset(array, identity_gwcs):
meta = {
'bucket': 'data',
'dataset_id': 'test_dataset',
'asdf_object_key': 'test_dataset.asdf'
}
ds = Dataset(array, wcs=identity_gwcs, meta=meta)
# Sanity checks
assert ds.data is array
assert ds.wcs is identity_gwcs
ds._array_container = DaskFITSArrayContainer([
ExternalArrayReference('test1.fits', 0, float, (10, 10)),
ExternalArrayReference('test2.fits', 0, float, (10, 10))
],
loader=AstropyFITSLoader)
return ds
def from_yaml_tree(self, node, tag, ctx):
from dkist.dataset import Dataset
data = node["data"]._generate_array()
wcs = node["wcs"]
meta = node.get("meta", {})
unit = node.get("unit")
mask = node.get("mask")
# Support older versions of the schema where headers was it's own top
# level property
if tag in ("tag:dkist.nso.edu:dkist/dataset-0.1.0",
"tag:dkist.nso.edu:dkist/dataset-0.2.0"):
meta["inventory"] = node.get("meta")
meta["headers"] = node["headers"]
dataset = Dataset(data, wcs=wcs, meta=meta, unit=unit, mask=mask)
dataset._file_manager = node["data"]
return dataset
def test_file_manager():
dataset = Dataset.from_directory(os.path.join(rootdir, 'EIT'))
assert dataset.files is dataset._file_manager
with pytest.raises(AttributeError):
dataset.files = 10
assert len(dataset.files.filenames) == 11
assert len(dataset.files.filenames) == 11
assert isinstance(dataset[5]._file_manager, FileManager)
assert len(dataset[..., 5].files.filenames) == 1
def main():
path = Path('~/sunpy/data/jsocflare/').expanduser()
files = glob.glob(str(path / '*.fits'))
# requestid = 'JSOC_20180831_1097'
requestid = None
if not files:
if requestid:
c = JSOCClient()
filesd = c.get_request(requestid, path=str(path),
overwrite=False).wait()
files = []
for f in filesd.values():
files.append(f['path'])
else:
results = Fido.search(
a.jsoc.Time('2017-09-06T12:00:00', '2017-09-06T12:02:00'),
a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Segment('image'),
a.jsoc.Notify("*****@*****.**"))
print(results)
files = Fido.fetch(results, path=str(path))
files.sort()
files = np.array(files)
# For each image get:
# the index
inds = []
# the time
times = []
# the dt from the first image
seconds = []
# the wavelength
waves = []
for i, filepath in enumerate(files):
with fits.open(filepath) as hdul:
header = hdul[1].header
time = parse_time(header['DATE-OBS'])
if i == 0:
root_header = header
start_time = time
inds.append(i)
times.append(time)
seconds.append((time - start_time).total_seconds())
waves.append(header['WAVELNTH'])
# Construct an array and sort it by wavelength and time
arr = np.array((inds, seconds, waves)).T
sorter = np.lexsort((arr[:, 1], arr[:, 2]))
# Using this double-sorted array get the list indicies
list_sorter = np.array(arr[sorter][:, 0], dtype=int)
# Calculate the desired shape of the output array
n_waves = len(list(set(waves)))
shape = (n_waves, len(files) // n_waves)
# Construct a 2D array of filenames
cube = files[list_sorter].reshape(shape)
# Extract a list of coordinates in time and wavelength
# this assumes all wavelength images are taken at the same time
time_coords = np.array([t.isoformat()
for t in times])[list_sorter].reshape(shape)[0, :]
wave_coords = np.array(waves)[list_sorter].reshape(shape)[:, 0]
smap0 = sunpy.map.Map(files[0])
spatial = map_to_transform(smap0)
timemodel = generate_lookup_table(lookup_table=seconds[:shape[1]] * u.s)
wavemodel = generate_lookup_table(lookup_table=waves[:shape[0]] * u.AA)
hcubemodel = spatial & timemodel & wavemodel
wave_frame = cf.SpectralFrame(axes_order=(3, ),
unit=u.AA,
name="wavelength",
axes_names=("wavelength", ))
time_frame = cf.TemporalFrame(axes_order=(2, ),
unit=u.s,
reference_time=Time(time_coords[0]),
name="time",
axes_names=("time", ))
sky_frame = cf.CelestialFrame(axes_order=(0, 1),
name='helioprojective',
reference_frame=smap0.coordinate_frame,
axes_names=("helioprojective longitude",
"helioprojective latitude"))
sky_frame = cf.CompositeFrame([sky_frame, time_frame, wave_frame])
detector_frame = cf.CoordinateFrame(name="detector",
naxes=4,
axes_order=(0, 1, 2, 3),
axes_type=("pixel", "pixel", "pixel",
"pixel"),
axes_names=("x", "y", "time",
"wavelength"),
unit=(u.pix, u.pix, u.pix, u.pix))
wcs = gwcs.wcs.WCS(forward_transform=hcubemodel,
input_frame=detector_frame,
output_frame=sky_frame)
print(repr(wcs))
print(wcs(*[1 * u.pix] * 4, with_units=True))
ea = references_from_filenames(cube, relative_to=str(path))
tree = {
'gwcs': wcs,
'dataset': ea,
}
with asdf.AsdfFile(tree) as ff:
# ff.write_to("test.asdf")
filename = str(path / "aia_{}.asdf".format(time_coords[0]))
ff.write_to(filename)
print("Saved to : {}".format(filename))
# import sys; sys.exit(0)
from dkist.dataset import Dataset
ds = Dataset.from_directory(str(path))
print(repr(ds))
print(repr(ds.wcs))
print(ds.wcs(*[1 * u.pix] * 4, with_units=True))
def test_from_directory_not_dir():
with pytest.raises(ValueError) as e:
Dataset.from_directory(rootdir / 'EIT' / 'eit_2004-03-01T00_00_10.515000.asdf')
assert "must be a directory" in str(e)
def test_from_not_directory():
with pytest.raises(ValueError) as e:
Dataset.from_directory(rootdir / "notadirectory")
assert "directory argument" in str(e)
def test_from_directory_no_asdf(tmpdir):
with pytest.raises(ValueError) as e:
Dataset.from_directory(tmpdir)
assert "No asdf file found" in str(e)
def test_load_from_directory():
ds = Dataset.from_directory(os.path.join(rootdir, 'EIT'))
assert isinstance(ds.data, da.Array)
assert isinstance(ds.wcs, gwcs.WCS)
assert_quantity_allclose(ds.dimensions, (11, 128, 128)*u.pix)
def test_load_invalid_asdf(invalid_asdf):
with pytest.raises(TypeError):
Dataset.from_asdf(invalid_asdf)
def dataset_4d(identity_gwcs_4d):
shape = (50, 60, 70, 80)
x = np.ones(shape)
array = da.from_array(x, tuple(shape))
return Dataset(array, wcs=identity_gwcs_4d)
def test_make_asdf(header_filenames, tmpdir):
path = pathlib.Path(header_filenames[0])
dataset_from_fits(path.parent, "test.asdf")
assert (path.parent / "test.asdf").exists()
assert isinstance(Dataset.from_directory(str(path.parent)), Dataset)
def test_from_directory_not_dir():
with pytest.raises(ValueError) as e:
Dataset.from_directory(os.path.join(rootdir, 'EIT', 'eit_2004-03-01T00:00:10.515000.asdf'))
assert "must be a directory" in str(e)