def load_pvl(input_pvl, decoder=None):
if isinstance(input_pvl, str):
try:
label = pvl.load(input_pvl, decoder=decoder)
except:
# Some labels are poorly formatted or include characters that
# cannot be parsed with the PVL library. This finds those
# characters and replaces them so that we can properly parse
# the PVL.
with open(input_pvl, 'r') as f:
filedata = f.read()
filedata = filedata.replace(';', '-').replace('&', '-')
filedata = re.sub(r'\-\s+', r'', filedata, flags=re.M)
with open(input_pvl, 'w') as f:
f.write(filedata)
label = pvl.load(input_pvl)
label = lower_keys(label)
elif isinstance(input_pvl, pvl.PVLModule):
label = lower_keys(input_pvl)
return label
def test_dump_stream():
for filename in PDS_LABELS:
label = pvl.load(filename)
stream = io.BytesIO()
pvl.dump(label, stream)
stream.seek(0)
assert label == pvl.load(stream)
def test_dump_stream():
for filename in PDS_LABELS:
label = pvl.load(filename)
stream = io.BytesIO()
pvl.dump(label, stream)
stream.seek(0)
assert label == pvl.load(stream)
def parse_label(filename, full=False):
""" Wraps forking paths for attached and detached PDS3 labels.
"""
if filename.endswith('.fmt'):
return pvl.load(filename)
if not has_attached_label(filename):
if os.path.exists(filename[:filename.rfind(".")] + ".LBL"):
label = pvl.load(filename[:filename.rfind(".")] + ".LBL")
elif os.path.exists(filename[:filename.rfind(".")] + ".lbl"):
label = pvl.load(filename[:filename.rfind(".")] + ".lbl")
elif os.path.exists(filename[:filename.rfind(".")] + ".xml"):
# TODO: Make label data format consistent between PDS3 & 4
label = pds4_tools.read(filename[:filename.rfind(".")] + ".xml",
quiet=True).label.to_dict()
else:
print("*** Unable to locate file label. ***")
return None
else:
label = parse_attached_label(filename)
# TODO: This ugly conditional exists entirely to deal with Cassini data
# which all seem to be returning zero-value images, so maybe it's wrong!
if (not full) and ("UNCOMPRESSED_FILE" in label.keys()):
if "COMPRESSED_FILE" in label.keys():
if "ENCODING_TYPE" in label["COMPRESSED_FILE"].keys():
if label["COMPRESSED_FILE"][
"ENCODING_TYPE"] == "MSLMMM-COMPRESSED":
return label
return label["UNCOMPRESSED_FILE"]
return label
class TestPancam(object):
pancam1 = Pancam(pvl.load(FILE_2))
pancam2 = Pancam(pvl.load(FILE_3))
def test_camera(self):
assert self.pancam1.camera == 'PANCAM_RIGHT'
assert self.pancam2.camera == 'PANCAM_LEFT'
def test_is_left(self):
assert not self.pancam1.is_left
assert self.pancam2.is_left
def test_is_right(self):
assert self.pancam1.is_right
assert not self.pancam2.is_right
def test_filter_num(self):
assert self.pancam1.filter_num == 8
assert self.pancam2.filter_num == 8
@pytest.mark.parametrize('unit, wavelength1, wavelength2',
[('nm', 880, 440), ('um', 0.880, 0.440),
('AA', 8800, 4400)])
def test_get_wavelength(self, unit, wavelength1, wavelength2):
assert self.pancam1.get_wavelength(unit) == wavelength1
assert self.pancam2.get_wavelength(unit) == wavelength2
def parse_attached_label(filename):
""" Parse an attached label of a IMG file.
"""
# First grab the entries from the label that define how to read the label
with open(filename, "rb") as f:
for line_ in f:
line = line_.decode("utf-8").strip() # hacks through a rare error
if "PDS_VERSION_ID" in line:
PDS_VERSION_ID = line.strip().split("=")[1]
if "RECORD_BYTES" in line:
RECORD_BYTES = int(line.strip().split("=")[1])
if "LABEL_RECORDS" in line:
if "<BYTES>" in line:
# Convert pointer value to bytes like everything else
LABEL_RECORDS = line.strip().split("=")[1].split()[0] * 8
else:
LABEL_RECORDS = int(line.strip().split("=")[1])
break
# Read the label and then parse it with PVL
try:
with open(filename, "rb") as f:
return pvl.load(f.read(RECORD_BYTES * (LABEL_RECORDS)))
except UnboundLocalError:
print("*** RECORD_BYTES not set??? ***")
return None
except:
with open(filename, "rb") as f:
return pvl.load(f.read(RECORD_BYTES * (LABEL_RECORDS)),
strict=False)
def main():
args = Args()
args.parse_args()
a = {}
if args.textfile is not None:
filePath = open(args.textfile, 'r')
lines = filePath.readlines()
length = len(lines)
for n in range(length):
if 'https' in lines[n] or 'http' in lines[n] or 'ftp' in lines[n]:
voldesc = urllib.request.urlopen(lines[n])
voldescPvl = pvl.load(voldesc)
else:
voldescPvl = load_pvl(lines[n].rstrip())
a.update(ds_count(voldescPvl))
else:
if 'https' in args.local or 'http' in args.local or 'ftp' in args.local:
voldesc = urllib.request.urlopen(args.local)
voldescPvl = pvl.load(voldesc)
else:
filePath = open(str(args.local), 'r')
voldescPvl = load_pvl(str(args.local))
a.update(ds_count(voldescPvl))
if args.output is not None:
f = open(args.output, 'w')
f.write(str(json.dumps(a)))
else:
print(json.dumps(a))
def __init__(self, path: os.PathLike):
# First try it as an ISIS cube:
try:
cam_to = Path(path).with_suffix(".caminfo")
sh.caminfo(f'from={path}', f'to={cam_to}', 'polygon=True')
caminfo = pvl.load(str(cam_to))
except subprocess.CalledProcessError:
# Maybe it is the PVL output of caminfo?
caminfo = pvl.load(str(path))
camgeo = caminfo["Caminfo"]["Geometry"]
self.path = Path(path)
self.incidence_angle = float(camgeo["IncidenceAngle"])
self.emission_angle = float(camgeo["EmissionAngle"])
self.phase_angle = float(camgeo["PhaseAngle"])
self.sc_azimuth = float(camgeo["SubSpacecraftGroundAzimuth"])
self.solar_azimuth = float(camgeo["SubSolarGroundAzimuth"])
self.gsd = float(camgeo["ObliquePixelResolution"])
try:
from shapely import wkt
self.geometry = wkt.loads(
caminfo["Caminfo"]["Polygon"]["GisFootprint"])
except ImportError:
self.geometry = None
def test_dump_to_file():
tmpdir = tempfile.mkdtemp()
try:
for filename in PDS_LABELS:
label = pvl.load(filename)
tmpfile = os.path.join(tmpdir, os.path.basename(filename))
pvl.dump(label, tmpfile)
assert label == pvl.load(tmpfile)
finally:
shutil.rmtree(tmpdir)
def test_dump_to_file():
tmpdir = tempfile.mkdtemp()
try:
for filename in PDS_COMPLIANT:
label = pvl.load(filename)
tmpfile = os.path.join(tmpdir, os.path.basename(filename))
pvl.dump(label, tmpfile)
assert label == pvl.load(tmpfile)
finally:
shutil.rmtree(tmpdir)
def test_broken_labels(label, expected, expected_errors):
with open(os.path.join(BROKEN_DIR, label), 'rb') as stream:
module = pvl.load(stream, strict=False)
expected = pvl.PVLModule(expected)
assert module == expected
assert module.errors == expected_errors
assert not module.valid
with open(os.path.join(BROKEN_DIR, label), 'rb') as stream:
with pytest.raises(pvl.decoder.ParseError):
pvl.load(stream, strict=True)
def test_broken_labels(label, expected, expected_errors):
with open(os.path.join(BROKEN_DIR, label), 'rb') as stream:
module = pvl.load(stream, strict=False)
expected = pvl.PVLModule(expected)
assert module == expected
assert module.errors == expected_errors
assert not module.valid
with open(os.path.join(BROKEN_DIR, label), 'rb') as stream:
with pytest.raises(pvl.decoder.ParseError):
pvl.load(stream, strict=True)
def test_dump_to_file_insert_before():
tmpdir = tempfile.mkdtemp()
try:
for filename in PDS_COMPLIANT:
label = pvl.load(filename)
if os.path.basename(filename) != 'empty.lbl':
label.insert_before('PDS_VERSION_ID', [('new', 'item')])
tmpfile = os.path.join(tmpdir, os.path.basename(filename))
pvl.dump(label, tmpfile, encoder=pvl.encoder.PVLEncoder())
assert label == pvl.load(tmpfile)
finally:
shutil.rmtree(tmpdir)
def test_dump_to_file_insert_after():
tmpdir = tempfile.mkdtemp()
try:
for filename in PDS_LABELS:
label = pvl.load(filename)
if os.path.basename(filename) != 'empty.lbl':
label.insert_after('PDS_VERSION_ID', [('new', 'item')])
tmpfile = os.path.join(tmpdir, os.path.basename(filename))
pvl.dump(label, tmpfile)
assert label == pvl.load(tmpfile)
finally:
shutil.rmtree(tmpdir)
def test_dump_to_file_insert_after():
tmpdir = tempfile.mkdtemp()
try:
for filename in PDS_LABELS:
label = pvl.load(filename)
if os.path.basename(filename) != 'empty.lbl':
label.insert_after('PDS_VERSION_ID', [('new', 'item')])
tmpfile = os.path.join(tmpdir, os.path.basename(filename))
pvl.dump(label, tmpfile)
assert label == pvl.load(tmpfile)
finally:
shutil.rmtree(tmpdir)
def test_dump_to_file_insert_after():
tmpdir = tempfile.mkdtemp()
try:
for filename in PDS_COMPLIANT:
label = pvl.load(filename)
if os.path.basename(filename) != "empty.lbl":
label.insert_after("PDS_VERSION_ID", [("new", "item")])
tmpfile = os.path.join(tmpdir, os.path.basename(filename))
pvl.dump(label, tmpfile, encoder=pvl.encoder.PVLEncoder())
assert label == pvl.load(tmpfile)
finally:
shutil.rmtree(tmpdir)
def __init__(
self,
# `uvis_id` can be either the PDS product_id 'FUV2005_172_03_35' or the long form
# used within the UVIS product.
# The attribute `pid` will carry the shortenend PDS identifier,
# attribute `uvis_id` will just store what the user came in with.
uvis_id: str,
skip_download: bool = False):
self.uvis_id = uvis_id
self.product_id = self.pid = uvis_id[:17]
self.path = get_data_path(self.pid, skip_download)
if self.path is None:
raise FileNotFoundError("No valid data path found.")
self.pds = PDSReader(self.path)
self.datalabel = self.pds.label
self.cal_data = None
# Try to load the calibration Matrix file. If not present, set to None.
if self.cal_label_path.exists() and self.cal_data_path.exists():
self.cal_data = PDSReader(self.cal_data_path)
self.cal_matrix = self.cal_data.data
self.caliblabel = dict2obj(pvl.load(str(self.cal_label_path)))
self.set_cal_wavelengths()
else:
self.set_default_wavelengths()
def filterCNetPVL(self, path=None):
"""Filters the CNET file and adds Ignored point information."""
if len(self.IgnoredPoints) == 0:
return
if path is None:
path = self.cnet_path
p = pvl.load(str(path))
cn = pvl.PVLModule()
badness = 0
for (k, v) in p["ControlNetwork"].items():
if k == "ControlPoint":
if (v["PointId"] in self.IgnoredPoints
and "Ignore" not in v.keys()):
v.append("Ignore", True)
badness += 1
logger.info("Ignoring point {}".format(v["PointId"]))
cn.append(k, v)
logger.info(f"{badness} point(s) ignored.")
new_pvl = pvl.PVLModule(ControlNetwork=cn)
with open(path, "w") as stream:
pvl.dump(new_pvl, stream, encoder=pvl.encoder.ISISEncoder())
def extract_img(image_file: str) -> Image:
"""Extracts and returns embedded image from PDS3 IMG files as PIL Image.
Args:
image_file (str): path to .IMG file.
Returns:
PIL.Image
"""
# Parsing label
label = pvl.load(image_file) # load label from .IMG file
image_data = label['IMAGE'] # getting image object info
h_img, w_img = image_data[0][-1], image_data[1][-1] # real image sizes
pref, suff = image_data[6][-1], image_data[7][-1] # buffer pixels margins
w_total = w_img + pref + suff # width with margins
offset = label['^IMAGE'].value # pointer where image is located
size = label['^GAP_TABLE'].value - label[
'^IMAGE'].value # image size (in bytes)
# Now getting back to file
with open(image_file, "rb") as f:
container = ContainerIO.ContainerIO(f, offset - 1, size)
data = container.read() # reading image bytes
img = Image.frombytes('L', (w_total, h_img), data, "raw") # decoding
img = img.crop((pref, 0, w_img + pref, h_img)) # cropping margins
return img
def object_asarray(file_path: os.PathLike, name: str) -> np.ndarray:
"""Return the data object of *name* from *file_path* as a numpy array.
Data objects are found by having both a a pointer parameter (like ^*name*)
and an object in the label with *name*.
Doesn't work for objects that have columns. Since such objects don't
have SAMPLE_BITS and SAMPLE_TYPE fields, this will result in a KeyError.
"""
label = pvl.load(str(file_path))
width, b_order, b_signed = byte_info(name, label)
lut = LUT_Table(
label["INSTRUMENT_SETTING_PARAMETERS"]["MRO:LOOKUP_CONVERSION_TABLE"])
table = list()
with open(file_path, mode="rb") as f:
# Since The first byte is location 1 (not 0) in PDS-counting.
f.seek(int(label[f"^{name}"].value) - 1)
for line in range(label[name]["LINES"]):
row = list()
# Skip over the prefix bytes.
f.seek(label[name]["LINE_PREFIX_BYTES"], os.SEEK_CUR)
for samp in range(label[name]["LINE_SAMPLES"]):
dn = lut.unlut(
int.from_bytes(f.read(width),
byteorder=b_order,
signed=b_signed))
row.append(dn)
table.append(row)
# Skip over the suffix bytes.
f.seek(label[name]["LINE_SUFFIX_BYTES"], os.SEEK_CUR)
return np.array(table)
def overwrite_object(file_path: os.PathLike, name: str, arr: np.ndarray):
"""The file at *file_path* will have its *name* object overwritten
with the data in *array*.
Doesn't work for objects that have columns. Since such objects don't
have SAMPLE_BITS and SAMPLE_TYPE fields, this will result in a KeyError.
"""
label = pvl.load(str(file_path))
width, b_order, b_signed = byte_info(name, label)
if arr.shape != (label[name]["LINES"], label[name]["LINE_SAMPLES"]):
raise ValueError(f"The object {name} has different dimensions "
f"({label[name]['LINES']}, "
f"{label[name]['LINE_SAMPLES']})than "
f"the array ({arr.shape}).")
lut = LUT_Table(
label["INSTRUMENT_SETTING_PARAMETERS"]["MRO:LOOKUP_CONVERSION_TABLE"])
with open(file_path, mode="r+b") as f:
# Since The first byte is location 1 (not 0) in PDS-counting.
f.seek(int(label[f"^{name}"].value) - 1)
for line in range(arr.shape[0]):
# Skip over the prefix bytes.
f.seek(label[name]["LINE_PREFIX_BYTES"], os.SEEK_CUR)
for samp in range(arr.shape[1]):
f.write(
lut.lookup(arr[line][samp]).to_bytes(width,
byteorder=b_order,
signed=b_signed))
# Skip over the suffix bytes.
f.seek(label[name]["LINE_SUFFIX_BYTES"], os.SEEK_CUR)
return
def readLBL(self):
'''Read VIMS LBL header'''
self.lbl = pvl.load(self.imgID + '.LBL')
for ii, axis in enumerate(self.lbl['QUBE']['AXIS_NAME']):
if axis == 'SAMPLE':
self.NS = int(self.lbl['QUBE']['CORE_ITEMS'][ii])
elif axis == 'LINE':
self.NL = int(self.lbl['QUBE']['CORE_ITEMS'][ii])
elif axis == 'BAND':
self.NB = int(self.lbl['QUBE']['CORE_ITEMS'][ii])
self.obs = self.lbl['INSTRUMENT_HOST_NAME']
self.inst = self.lbl['INSTRUMENT_ID']
self.target = self.lbl['TARGET_NAME']
self.expo = self.lbl['FRAME_PARAMETER'][0]
#self.mode = self.lbl['QUBE']['SAMPLING_MODE_ID'][0]
#self.seq = self.lbl['QUBE']['SEQUENCE_ID']
#self.seq_title = self.lbl['QUBE']['SEQUENCE_TITLE']
self.start = self.lbl['START_TIME']
self.stop = self.lbl['STOP_TIME']
self.dtime = (self.stop - self.start) / 2 + self.start
self.time = self.dtime.strftime('%Y-%m-%dT%H:%M:%S.%f')
self.year = self.dtime.year
self.doy = int(self.dtime.strftime('%j'))
self.year_d = self.year + (
self.doy - 1
) / 365. # Decimal year [ISSUE: does not apply take into account bissextile years]
self.date = self.dtime.strftime('%Y/%m/%d')
self.wvlns = np.array(self.lbl['QUBE']['BAND_BIN']['BAND_BIN_CENTER'])
self.bands = np.array(
self.lbl['QUBE']['BAND_BIN']['BAND_BIN_ORIGINAL_BAND'])
return
def main():
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
if len(args.img) > 1 and not args.output.startswith("."):
logger.critical(
"With more than one input IMG file, the --output must start with "
f"a period, and it does not: {args.output}"
)
sys.exit()
gainsinfo = pvl.load(args.gains)
for i in args.img:
out_p = util.path_w_suffix(args.output, i)
with util.main_exceptions(args.verbose):
histats = EDR_Stats(
i, out_p, gainsinfo, args.histmin, args.histmax, keep=args.keep
)
# DB stuff
# for k, v in histats.items():
# print(f'{k}: {v}')
db_path = util.path_w_suffix(args.db, i)
with open(db_path, "w") as f:
json.dump(histats, f, indent=0, sort_keys=True)
logger.info(f"Wrote {db_path}")
return
def readLBL(self):
'''Read VIMS LBL header'''
self.lbl = pvl.load(self.fname)
for ii, axis in enumerate(self.lbl['SPECTRAL_QUBE']['AXIS_NAME']):
if axis == 'SAMPLE':
self.NS = int(self.lbl['SPECTRAL_QUBE']['CORE_ITEMS'][ii])
elif axis == 'LINE':
self.NL = int(self.lbl['SPECTRAL_QUBE']['CORE_ITEMS'][ii])
elif axis == 'BAND':
self.NB = int(self.lbl['SPECTRAL_QUBE']['CORE_ITEMS'][ii])
self.obs = self.lbl['INSTRUMENT_HOST_NAME']
self.inst = self.lbl['INSTRUMENT_ID']
self.target = self.lbl['TARGET_NAME']
self.expo = {'IR': self.lbl['EXPOSURE_DURATION'][0], 'VIS': self.lbl['EXPOSURE_DURATION'][1]}
self.mode = {'IR': self.lbl['SAMPLING_MODE_ID'][0], 'VIS': self.lbl['SAMPLING_MODE_ID'][1]}
self.seq = self.lbl['SEQUENCE_ID']
self.seq_title = self.lbl['SEQUENCE_TITLE']
self.start = self.lbl['START_TIME']
self.stop = self.lbl['STOP_TIME']
self.dtime = (self.stop - self.start)/2 + self.start
self.time = self.dtime.strftime('%Y-%m-%dT%H:%M:%S.%f')
self.year = self.dtime.year
self.doy = int(self.dtime.strftime('%j'))
self.year_d = self.year + (self.doy-1)/365. # Decimal year [ISSUE: doest not apply take into account bissextile years]
self.date = self.dtime.strftime('%Y/%m/%d')
self.wvlns = None
self.bands = None
return
def parse_label(label, grammar=pvl.grammar.PVLGrammar):
"""
Attempt to parse a PVL label.
Parameters
----------
label
The label as a pvl string or pvl file.
grammar
The pvl grammar with which to parse the label. If None, default to PVLGrammar
Returns
-------
pvl.collections.PVLModule
The PVL label deserialized to a Python object
See Also
--------
load
loads
"""
try:
parsed_label = pvl.loads(label, grammar=grammar)
except Exception:
parsed_label = pvl.load(label, grammar=grammar)
except:
raise ValueError("{} is not a valid label for grammar {}".format(label, grammar.__name__))
return parsed_label
def extract_band(job, image, band_num):
"""
Extract the temperature data from the processed ISIS cube.
Parameters
----------
job : dict
Job specification dictionary
image : str
PATH to an ISIS cube to extract bands from
band_num : int
Band number that needs to be extracted
Returns
----------
: ndarray
Array representation of the extracted band
"""
header = pvl.load(job['images'])
bands = find_in_dict(header, 'BAND_BIN_BAND_NUMBER')
for i, band in enumerate(bands):
if band_num == band:
geo_image = io_gdal.GeoDataset(image)
return geo_image.read_array(band=i + 1)
def label(self):
"""
Loads a PVL from from the _file attribute and
parses the binary table data.
Returns
-------
PVLModule :
Dict-like object with PVL keys
"""
class PvlDecoder(pvl.decoder.PVLDecoder):
def unescape_next_char(self, stream):
esc = stream.read(1)
string = '\{}'.format(esc.decode('utf-8')).encode('utf-8')
return string
if not hasattr(self, "_label"):
if isinstance(self._file, pvl.PVLModule):
self._label = self._file
try:
self._label = pvl.loads(self._file, PvlDecoder)
except Exception:
# PvlDecoder class to ignore all escape sequences when getting
# the label
self._label = pvl.load(self._file, PvlDecoder)
except:
raise ValueError("{} is not a valid label".format(self._file))
return self._label
def metadata(self):
if not hasattr(self, '_metadata'):
try:
self._metadata = pvl.load(self.file_name)
except:
self._metadata = self.dataset.GetMetadata()
return self._metadata
def main():
# The original Perl needed the specific output of cubenorm from a
# particular step in the HiCal pipeline before here. However, rather
# than keep track of those files, open them, and read them again, we
# can (and did) perform the relevant check in HiCal.py, and then save
# that in the db that we can check now.
args = arg_parser().parse_args()
util.set_logger(args.verbose, args.logfile, args.log)
with util.main_exceptions(args.verbose):
(db, outcub_path) = HiStitch(
args.cube0,
args.cube1,
get_db(util.pid_path_w_suffix(args.db, args.cube0)),
get_db(util.pid_path_w_suffix(args.db2, args.cube1)),
args.output,
pvl.load(args.conf),
keep=args.keep,
)
db_path = set_outpath(args.dbout, hirise.get_CCDID_fromfile(outcub_path),
outcub_path.parent)
with open(db_path, "w") as f:
json.dump(db, f, indent=0, sort_keys=True)
logger.info(f"Wrote {db_path}")
return
def main():
args = arg_parser(formats).parse_args()
some_pvl = pvl.load(args.infile)
formats[args.output_format].dump(some_pvl, args.outfile)
return
def metadata(self):
if not hasattr(self, '_metadata'):
try:
self._metadata = pvl.load(self.file_name)
except:
self._metadata = self.dataset.GetMetadata()
return self._metadata
def __init__(self, fname):
self.path = Path(fname)
# file management
self.file_id = self.path.stem
self.label_fname = self.path.with_suffix(".LBL")
self.data_fname = self.path.with_suffix(".DAT")
# read the data
self.data1D = (np.fromfile(str(self.data_fname), ">H")).astype(np.uint16)
# label stuff
self.label = pvl.load(str(self.label_fname))
self.cubelabel = self.label["QUBE"]
self.LINE_BIN = self.cubelabel["LINE_BIN"]
self.BAND_BIN = self.cubelabel["BAND_BIN"]
self.shape = tuple(self.cubelabel["CORE_ITEMS"])
self.line_range = (
self.cubelabel["UL_CORNER_LINE"],
self.cubelabel["LR_CORNER_LINE"] + 1, # for numpy slicing + 1
)
self.band_range = (
self.cubelabel["UL_CORNER_BAND"],
self.cubelabel["LR_CORNER_BAND"] + 1, # for numpy slicing + 1
)
# reshape the data with infos from label
self.data = self.data1D.reshape(self.shape, order="F")[
slice(*self.band_range), slice(*self.line_range), :
]
def read(self):
"""
Given an ISIS store, read the underlying ISIS3 compatible control network and
return an IsisControlNetwork dataframe.
"""
pvl_header = pvl.load(self._path)
header_start_byte = find_in_dict(pvl_header, 'HeaderStartByte')
header_bytes = find_in_dict(pvl_header, 'HeaderBytes')
point_start_byte = find_in_dict(pvl_header, 'PointsStartByte')
version = find_in_dict(pvl_header, 'Version')
if version == 2:
self.point_attrs = [i for i in cnf._CONTROLPOINTFILEENTRYV0002.fields_by_name if i != 'measures']
self.measure_attrs = [i for i in cnf._CONTROLPOINTFILEENTRYV0002_MEASURE.fields_by_name]
cp = cnf.ControlPointFileEntryV0002()
self._handle.seek(header_start_byte)
pbuf_header = cnf.ControlNetFileHeaderV0002()
pbuf_header.ParseFromString(self._handle.read(header_bytes))
self._handle.seek(point_start_byte)
cp = cnf.ControlPointFileEntryV0002()
pts = []
for s in pbuf_header.pointMessageSizes:
cp.ParseFromString(self._handle.read(s))
pt = [getattr(cp, i) for i in self.point_attrs if i != 'measures']
for measure in cp.measures:
meas = pt + [getattr(measure, j) for j in self.measure_attrs]
pts.append(meas)
elif version == 5:
self.point_attrs = [i for i in cnp5._CONTROLPOINTFILEENTRYV0005.fields_by_name if i != 'measures']
self.measure_attrs = [i for i in cnp5._CONTROLPOINTFILEENTRYV0005_MEASURE.fields_by_name]
cp = cnp5.ControlPointFileEntryV0005()
self._handle.seek(header_start_byte)
pbuf_header = cnh5.ControlNetFileHeaderV0005()
pbuf_header.ParseFromString(self._handle.read(header_bytes))
self._handle.seek(point_start_byte)
cp = cnp5.ControlPointFileEntryV0005()
pts = []
byte_count = 0;
while byte_count < find_in_dict(pvl_header, 'PointsBytes'):
message_size = struct.unpack('I', self._handle.read(4))[0]
cp.ParseFromString(self._handle.read(message_size))
pt = [getattr(cp, i) for i in self.point_attrs if i != 'measures']
for measure in cp.measures:
meas = pt + [getattr(measure, j) for j in self.measure_attrs]
pts.append(meas)
byte_count += 4 + message_size
cols = self.point_attrs + self.measure_attrs
df = IsisControlNetwork(pts, columns=cols)
df.header = pvl_header
return df
def read_label(filename):
try:
label = pvl.load(filename, strict=False)
if not len(label):
raise ValueError("Cannot find attached label data.")
label.append("attached_label_filename", filename)
return label
except: # look for a detached label
if os.path.exists(filename[:filename.rfind(".")] + ".LBL"):
return pvl.load(filename[:filename.rfind(".")] + ".LBL",
strict=False)
elif os.path.exists(filename[:filename.rfind(".")] + ".lbl"):
return pvl.load(filename[:filename.rfind(".")] + ".lbl",
strict=False)
else:
print(" *** Cannot find label data. *** ")
raise
def test_load_all_sample_labels():
files = glob.glob(os.path.join(PDS_DATA_DIR, "*.lbl"))
for infile in files:
try:
label = pvl.load(infile)
except:
raise
assert isinstance(label, Label)
def get_campt_label(frompath, sample, line):
try:
group = pvl.load(campt(from_=str(frompath), sample=sample,
line=line)).get('GroundPoint')
except ProcessError as e:
print(e.stdout)
print(e.stderr)
raise e
else:
return group
def get_rdr_index_names():
lblfile = hirise_dropbox() / 'RDRCUMINDEX.LBL'
label = pvl.load(lblfile)
table = label['RDR_INDEX_TABLE']
names = []
for item in table:
second = item[1]
if isinstance(second, pvl.PVLObject):
names.append(second['NAME'])
return names
def read_image_header(filename): # ^IMAGE_HEADER
label = parse_label(filename)
try:
with open(filename, "rb") as f:
f.seek(data_start_byte(label, "^IMAGE_HEADER"))
image_header = pvl.load(f.read(label["IMAGE_HEADER"]["BYTES"]))
return image_header
except:
print("*** Unable to parse image header. ***")
return
def get_rdr_index_names():
lblfile = hirise_dropbox() / 'RDRCUMINDEX.LBL'
label = pvl.load(str(lblfile))
table = label['RDR_INDEX_TABLE']
names = []
for item in table:
second = item[1]
if isinstance(second, pvl.PVLObject):
names.append(second['NAME'])
return names
def get_campt_label(frompath, sample, line):
try:
group = pvl.load(campt(from_=str(frompath),
sample=sample,
line=line)).get('GroundPoint')
except ProcessError as e:
print(e.stdout)
print(e.stderr)
raise e
else:
return group
def test_parse_error():
with pytest.raises(pvl.decoder.ParseError):
pvl.loads(b'foo=')
with pytest.raises(pvl.decoder.ParseError):
pvl.loads(b'=')
with pytest.raises(pvl.decoder.ParseError):
pvl.loads(b'(}')
with pytest.raises(pvl.decoder.ParseError):
pvl.loads(b'foo=')
with pytest.raises(pvl.decoder.ParseError):
pvl.loads(b'foo=!')
with pytest.raises(pvl.decoder.ParseError):
pvl.loads(b'foo')
with pytest.raises(pvl.decoder.ParseError):
pvl.load(io.BytesIO(b'foo'))
def test_create_pvl_header(self):
pvl_header = pvl.load("test.net")
npoints = find_in_dict(pvl_header, "NumberOfPoints")
self.assertEqual(2, npoints)
mpoints = find_in_dict(pvl_header, "NumberOfMeasures")
self.assertEqual(5, mpoints)
points_bytes = find_in_dict(pvl_header, "PointsBytes")
self.assertEqual(334, points_bytes)
points_start_byte = find_in_dict(pvl_header, "PointsStartByte")
self.assertEqual(65621, points_start_byte)
def test_create_pvl_header(self):
pvl_header = pvl.load('test.net')
npoints = find_in_dict(pvl_header, 'NumberOfPoints')
self.assertEqual(2, npoints)
mpoints = find_in_dict(pvl_header, 'NumberOfMeasures')
self.assertEqual(5, mpoints)
points_bytes = find_in_dict(pvl_header, 'PointsBytes')
self.assertEqual(330, points_bytes)
points_start_byte = find_in_dict(pvl_header, 'PointsStartByte')
self.assertEqual(65621, points_start_byte)
def generate_serial_number(label):
"""
Generate an ISIS compatible serial number using the ISIS
team's translation files
Parameters
----------
label : dict or str
A PVLModule object (dict) or string PATH
to a file containing a PVL header
Returns
-------
: str
The ISIS compatible serial number
"""
if not isinstance(label, PVLModule):
label = pvl.load(label, cls=SerialNumberDecoder)
# Get the translation information
translation = get_isis_translation(label)
if not translation:
warnings.warn('Unable to load an appropriate image translation.')
return
serial_number = []
# Sort the keys to ensure proper iteration order
keys = sorted(translation.keys())
for k in keys:
try:
group = translation[k]
search_key = group['InputKey']
search_position = group['InputPosition']
search_translation = {group['Translation'][1]:group['Translation'][0]}
sub_group = find_nested_in_dict(label, search_position)
serial_entry = sub_group[search_key]
if serial_entry in search_translation.keys():
serial_entry = search_translation[serial_entry]
elif '*' in search_translation.keys() and search_translation['*'] != '*':
serial_entry = search_translation['*']
serial_number.append(serial_entry)
except:
pass
return '/'.join(serial_number)
def extract_keywords(header, *args):
"""
For a given header, find all of the keys and return an unnested dict.
"""
try:
header = pvl.load(header)
except:
header = pvl.loads(header)
res = {}
# Iterate through all of the requested keys
for a in args:
try:
res[a] = find_in_dict(a)
except:
res[a] = None
return res
def get_isis_translation(label):
"""
Compute the ISIS serial number for a given image using
the input cube or the label extracted from the cube.
Parameters
----------
label : dict or str
A PVL dict object or file name to extract
the PVL object from
Returns
-------
translation : dict
A PVLModule object containing the extracted
translation file
"""
# Instantiate a DB session if not already instantiated
if not hasattr(plio, 'data_session'):
print(get_data('data.db'))
plio.data_session = setup_db_session(get_data('data.db'))
# Grab the label is not already read
if not isinstance(label, PVLModule):
label = pvl.load(label)
# Grab the spacecraft name and run it through the ISIS lookup
spacecraft_name = find_in_dict(label, 'SpacecraftName')
for row in plio.data_session.query(StringToMission).filter(StringToMission.key==spacecraft_name):
spacecraft_name = row.value.lower()
# Try and pull an instrument identifier
try:
instrumentid = find_in_dict(label, 'InstrumentId').capitalize()
except:
instrumentid = None
translation = None
# Grab the translation PVL object using the lookup
for row in plio.data_session.query(Translations).filter(Translations.mission==spacecraft_name,
Translations.instrument==instrumentid):
# Convert the JSON back to a PVL object
translation = PVLModule(row.translation)
return translation
def test_pds3_sample_image():
infile = os.path.join(PDS_DATA_DIR, "simple_image_1.lbl")
label = pvl.load(infile)
assert label['RECORD_TYPE'] == 'FIXED_LENGTH'
assert label['RECORD_BYTES'] == 824
assert label['LABEL_RECORDS'] == 1
assert label['FILE_RECORDS'] == 601
assert label['IMAGE']['LINES'] == 600
assert label['IMAGE']['LINE_SAMPLES'] == 824
image_group = label['IMAGE']
assert image_group['SAMPLE_TYPE'] == 'MSB_INTEGER'
assert image_group['SAMPLE_BITS'] == 8
assert abs(image_group['MEAN'] - 51.6778539644) <= 0.00001
assert image_group['MEDIAN'] == 50.0
assert image_group['MINIMUM'] == 0
assert image_group['MAXIMUM'] == 255
assert image_group['STANDARD_DEVIATION'] == 16.97019
assert image_group['CHECKSUM'] == 25549531
def test_cube_label():
with open(os.path.join(DATA_DIR, 'pattern.cub'), 'rb') as fp:
label = pvl.load(fp)
assert isinstance(label['Label'], dict)
assert label['Label']['Bytes'] == 65536
assert isinstance(label['IsisCube'], dict)
assert isinstance(label['IsisCube']['Core'], dict)
assert label['IsisCube']['Core']['StartByte'] == 65537
assert label['IsisCube']['Core']['Format'] == 'Tile'
assert label['IsisCube']['Core']['TileSamples'] == 128
assert label['IsisCube']['Core']['TileLines'] == 128
assert isinstance(label['IsisCube']['Core']['Dimensions'], dict)
assert label['IsisCube']['Core']['Dimensions']['Samples'] == 90
assert label['IsisCube']['Core']['Dimensions']['Lines'] == 90
assert label['IsisCube']['Core']['Dimensions']['Bands'] == 1
assert isinstance(label['IsisCube']['Core']['Pixels'], dict)
assert label['IsisCube']['Core']['Pixels']['Type'] == 'Real'
assert label['IsisCube']['Core']['Pixels']['ByteOrder'] == 'Lsb'
assert label['IsisCube']['Core']['Pixels']['Base'] == 0.0
assert label['IsisCube']['Core']['Pixels']['Multiplier'] == 1.0
def main( argv = None ):
inputlbl = None
outputConfig = None
run = None
template = None
if argv is None:
argv = sys.argv
argv = gdal.GeneralCmdLineProcessor( argv )
if argv is None:
return 1
nArgc = len(argv)
#/* -------------------------------------------------------------------- */
#/* Parse arguments. */
#/* -------------------------------------------------------------------- */
i = 1
while i < nArgc:
if EQUAL(argv[i], '-run'):
run = True
elif EQUAL(argv[i], '-template'):
i = i + 1
template = argv[i]
elif inputlbl is None:
inputlbl = argv[i]
elif outputConfig is None:
outputConfig = argv[i]
else:
return Usage(argv[0])
i = i + 1
if inputlbl is None:
return Usage(argv[0])
if outputConfig is None:
return Usage(argv[0])
if template is None:
template = 'pds4_template.xml'
#load ISIS3 label using PVL library
isis3lbl = pvl.load(inputlbl)
#open output config file
fileConfig = open(outputConfig, 'w')
print('writing {}'.format(outputConfig))
#Write first comment line
theLine = '#{0} {1} {2}\n'.format(sys.argv[0], sys.argv[1], sys.argv[2])
fileConfig.write(theLine)
#Next lines are not available in ISIS3 label
theLine = '-co VAR_TARGET_TYPE=Satellite\n'
fileConfig.write(theLine)
theLine = '-co VAR_INVESTIGATION_AREA_LID_REFERENCE="urn:nasa:pds:context:instrument_host:spacecraft.lro"\n'
fileConfig.write(theLine)
try:
target = (isis3lbl['IsisCube']['Mapping']['TargetName']).upper()
theLine = '-co VAR_TARGET={}\n'.format(target)
fileConfig.write(theLine)
except KeyError:
print('No Target in ISIS3 Label')
try:
mission = (isis3lbl['IsisCube']['Archive']['InstrumentHostName']).upper()
theLine = '-co VAR_INVESTIGATION_AREA_NAME="{}"\n'.format(mission)
fileConfig.write(theLine)
except KeyError:
print('No InstrumentHostName in ISIS3 Label')
try:
dataSetID = isis3lbl['IsisCube']['Archive']['DataSetId']
theLine = '-co VAR_LOGICAL_IDENTIFIER={}\n'.format(dataSetID)
fileConfig.write(theLine)
except KeyError:
print('No DataSetId in ISIS3 Label')
try:
observeID = isis3lbl['IsisCube']['Archive']['InstrumentId']
theLine = '-co VAR_OBSERVING_SYSTEM_NAME={}\n'.format(observeID)
fileConfig.write(theLine)
except KeyError:
print('No InstrumentId in ISIS3 Label')
try:
fileName = isis3lbl['IsisCube']['Archive']['ProductId']
theLine = '-co VAR_TITLE={}\n'.format(fileName)
fileConfig.write(theLine)
except KeyError:
print('No ProductId in ISIS3 Label')
#try:
#producerName = isis3lbl['IsisCube']['Archive']['ProducerFullName']
#except KeyError:
#print('No ProducerFullName in ISIS3 Label')
#try:
#proInstName = isis3lbl['IsisCube']['Archive']['ProducerInstitutionName']
#except KeyError:
#print('No ProducerInstitutionName in ISIS3 Label')
fileConfig.close()
#write out helper line for gdal - can run from here too
outPDS4 = inputlbl.replace('.cub','_pds4.xml')
theCmd='gdal_translate -of PDS4 -co IMAGE_FORMAT=GEOTIFF -co TEMPLATE={0} --optfile {1} {2} {3}'.format(template, outputConfig, inputlbl, outPDS4)
if run is None:
print('\nRecommended gdal run:')
print('{}\n'.format(theCmd))
else: #run gdal
os.system(theCmd)
def test_default_encoder():
for filename in PDS_LABELS:
label = pvl.load(filename)
assert label == pvl.loads(pvl.dumps(label))
def test_pds_encoder():
for filename in PDS_LABELS:
label = pvl.load(filename)
encoder = pvl.encoder.PDSLabelEncoder
assert label == pvl.loads(pvl.dumps(label, cls=encoder))
def __init__(self, input_data, cleaned=True, qa_threshold=2000):
"""
Read the .spc file, parse the label, and extract the spectra
Parameters
----------
input_data : string
The PATH to the input .spc file
cleaned : boolean
If True, mask the data based on the QA array.
"""
label_dtype_map = {'IEEE_REAL':'f',
'MSB_INTEGER':'i',
'MSB_UNSIGNED_INTEGER':'u'}
label = pvl.load(input_data)
self._label = label
with open(input_data, 'rb') as indata:
# Get the offsets
ancillary_data_offset = find_in_dict(label,
"^ANCILLARY_AND_SUPPLEMENT_DATA").value
wavelength_offset = find_in_dict(label, "^SP_SPECTRUM_WAV").value
raw_offset = find_in_dict(label, "^SP_SPECTRUM_RAW").value
ref2_offset = find_in_dict(label, "^SP_SPECTRUM_REF2").value
radiance_offset = find_in_dict(label, "^SP_SPECTRUM_RAD").value
ref1_offset = find_in_dict(label, "^SP_SPECTRUM_REF1").value
qa_offset = find_in_dict(label, "^SP_SPECTRUM_QA").value
l2d_offset = find_in_dict(label, "^L2D_RESULT_ARRAY").value
ancillary_data = find_in_dict(label, "ANCILLARY_AND_SUPPLEMENT_DATA")
nrows = ancillary_data['ROWS']
ncols = ancillary_data['COLUMNS']
rowbytes = ancillary_data['ROW_BYTES']
columns = []
bytelengths = []
datatypes = []
index = np.arange(nrows)
indata.seek(ancillary_data_offset - 1)
for i in ancillary_data.items():
if i[0] == 'COLUMN':
entry = i[1]
columns.append(str(entry['NAME']))
datatypes.append(label_dtype_map[entry['DATA_TYPE']])
bytelengths.append(entry['BYTES'])
strbytes = map(str, bytelengths)
rowdtype = list(zip(columns, map(''.join, zip(['>'] * ncols, datatypes, strbytes))))
d = np.fromstring(indata.read(rowbytes * nrows), dtype=rowdtype,
count=nrows)
self.ancillary_data = pd.DataFrame(d, columns=columns,
index=np.arange(nrows))
"""
print len(columns)
for i in range(nrows):
d = np.fromstring(indata.read(rowbytes), dtype=rowdtype, count=1)
self.ancillary_data.iloc[i] = d[0]
"""
assert(ncols == len(columns))
keys = ["SP_SPECTRUM_WAV","SP_SPECTRUM_RAW", "SP_SPECTRUM_REF1",
"SP_SPECTRUM_REF2", "SP_SPECTRUM_RAD", "SP_SPECTRUM_QA"]
array_offsets = [wavelength_offset, raw_offset, ref1_offset,
ref2_offset, radiance_offset, qa_offset]
offsets = dict(zip(keys, array_offsets))
arrays = {}
for k, offset in offsets.items():
indata.seek(offset - 1)
newk = k.split('_')[-1]
d = find_in_dict(label, k)
unit = d['UNIT']
lines = d['LINES']
scaling_factor = d['SCALING_FACTOR']
arr = np.fromstring(indata.read(lines * 296*2), dtype='>H').astype(np.float64)
arr = arr.reshape(lines, -1)
if isinstance(scaling_factor, float):
arr *= scaling_factor
arrays[newk] = arr
self.wavelengths = pd.Series(arrays['WAV'][0])
self.spectra = {}
for i in range(nrows):
self.spectra[i] = pd.DataFrame(index = self.wavelengths,
columns = ["Raw", "Highlands",
"Mare", "Radiance", "QA"])
self.spectra[i]['Raw'] = arrays['RAW'][i]
self.spectra[i]['Highlands'] = arrays['REF1'][i]
self.spectra[i]['Mare'] = arrays['REF2'][i]
self.spectra[i]['Radiance'] = arrays['RAD'][i]
self.spectra[i]['QA'] = arrays['QA'][i]
#self.spectra[i] = pd.concat([raw, high, self.mare, rad, qa], axis=1)
#self.spectra[i] = pd.concat([raw, high, self.mare, rad, qa], axis=1)
if cleaned:
self.spectra[i] = self.spectra[i][self.spectra[i]['QA'] < qa_threshold]
self.spectra[i] = Spectra(self.spectra[i])
# * Author: David P. Mayer, [email protected] # * # * License: Public Domain # * # ****************************************************************************** import sys import re import fileinput import pvl from pvl._collections import Units as PUnits inputlbl = sys.argv[1] print(inputlbl) isis3lbl = pvl.load(inputlbl) # ## Debug only ## # # Load the ISIS3 input from a file. Can be either detached label, or a cube with attached label # # isis3lbl = pvl.load('Mercury_MESSENGER_mosaic_global_250m_2013.lbl') # # isis3lbl = pvl.load('Lunar_Kaguya_MIMap_MineralDeconv_AbundanceSMFe_50N50S.lbl') # ################ # # TODO: Check that input contains some minimum required set of keywords before translating # Some keyword value pairs that should always appear at the top of the PDS3 label # The value of LABEL_REVISION_NOTE should be modified to suit the user's needs pds3toplevel = """PDS_VERSION_ID = PDS3 DD_VERSION_ID = PDSCAT1R100 LABEL_REVISION_NOTE = \"2017-05-01, David P. Mayer (USGS), initial PDS3 label pointing to GeoTIFF;\" RECORD_TYPE = FIXED_LENGTH
def __init__(self, input_data, cleaned=True, qa_threshold=2000):
"""
Read the .spc file, parse the label, and extract the spectra
Parameters
----------
input_data : string
The PATH to the input .spc file
cleaned : boolean
If True, mask the data based on the QA array.
nspectra : int
The number of spectra in the given data file
qa_threshold : int
The threshold value over which observations are masked as noise
if cleaned is True.
"""
label_dtype_map = {'IEEE_REAL':'f',
'MSB_INTEGER':'i',
'MSB_UNSIGNED_INTEGER':'u'}
label = pvl.load(input_data)
self.label = label
with open(input_data, 'rb') as indata:
# Extract and handle the ancillary data
ancillary_data = find_in_dict(label, "ANCILLARY_AND_SUPPLEMENT_DATA")
self.nspectra = nrows = ancillary_data['ROWS']
ncols = ancillary_data['COLUMNS']
rowbytes = ancillary_data['ROW_BYTES']
columns = []
bytelengths = []
datatypes = []
ancillary_data_offset = find_in_dict(label, "^ANCILLARY_AND_SUPPLEMENT_DATA").value
indata.seek(ancillary_data_offset - 1)
for i in ancillary_data.items():
if i[0] == 'COLUMN':
entry = i[1]
# Level 2B2 PVL has entries with 0 bytes, e.g. omitted.
if entry['BYTES'] > 0:
columns.append(str(entry['NAME']))
datatypes.append(label_dtype_map[entry['DATA_TYPE']])
bytelengths.append(entry['BYTES'])
else:
ncols -= 1
strbytes = map(str, bytelengths)
rowdtype = list(zip(columns, map(''.join, zip(['>'] * ncols, datatypes, strbytes))))
d = np.fromstring(indata.read(rowbytes * nrows), dtype=rowdtype,
count=nrows)
self.ancillary_data = pd.DataFrame(d, columns=columns,
index=np.arange(nrows))
assert(ncols == len(columns))
keys = []
array_offsets = []
for d in ['WAV', 'RAW', 'REF', 'REF1', 'REF2', 'DAR', 'QA']:
search_key = '^SP_SPECTRUM_{}'.format(d)
result = find_in_dict(label, search_key)
if result:
array_offsets.append(result.value)
keys.append('SP_SPECTRUM_{}'.format(d))
offsets = dict(zip(keys, array_offsets))
arrays = {}
for k, offset in offsets.items():
indata.seek(offset - 1)
newk = k.split('_')[-1]
d = find_in_dict(label, k)
unit = d['UNIT']
lines = d['LINES']
scaling_factor = d['SCALING_FACTOR']
arr = np.fromstring(indata.read(lines * 296*2), dtype='>H').astype(np.float64)
arr = arr.reshape(lines, -1)
# If the data is scaled, apply the scaling factor
if isinstance(scaling_factor, float):
arr *= scaling_factor
arrays[newk] = arr
self.wavelengths = pd.Series(arrays['WAV'][0])
self.spectra = {}
for i in range(nrows):
self.spectra[i] = pd.DataFrame(index=self.wavelengths)
for k in keys:
k = k.split('_')[-1]
if k == 'WAV':
continue
self.spectra[i][k] = arrays[k][i]
if cleaned:
self.spectra[i] = self.spectra[i][self.spectra[i]['QA'] < qa_threshold]
self.spectra = pd.Panel(self.spectra)
def __init__(self, input_data, label=None, cleaned=True, qa_threshold=2000):
"""
Read the .spc file, parse the label, and extract the spectra
Parameters
----------
input_data : string
The PATH to the input .spc file
label : string
The PATH to an optional detached label associated with the .spc
cleaned : boolean
If True, mask the data based on the QA array.
nspectra : int
The number of spectra in the given data file
qa_threshold : int
The threshold value over which observations are masked as noise
if cleaned is True.
"""
label_dtype_map = {'IEEE_REAL':'f',
'MSB_INTEGER':'i',
'MSB_UNSIGNED_INTEGER':'u'}
if label:
label = pvl.load(label)
else:
label = pvl.load(input_data)
self.label = label
self.input_data = input_data
with open(input_data, 'rb') as indata:
# Extract and handle the ancillary data
ancillary_data = find_in_dict(label, "ANCILLARY_AND_SUPPLEMENT_DATA")
self.nspectra = nrows = ancillary_data['ROWS']
ncols = ancillary_data['COLUMNS']
rowbytes = ancillary_data['ROW_BYTES']
columns = []
bytelengths = []
datatypes = []
try:
ancillary_data_offset = find_in_dict(self.label, "^ANCILLARY_AND_SUPPLEMENT_DATA").value
except:
ancillary_data_offset = find_in_dict(self.label, "^ANCILLARY_AND_SUPPLEMENT_DATA")[1].value
indata.seek(ancillary_data_offset - 1)
for i in ancillary_data.items():
if i[0] == 'COLUMN':
entry = i[1]
# Level 2B2 PVL has entries with 0 bytes, e.g. omitted.
if entry['BYTES'] > 0:
columns.append(str(entry['NAME']))
datatypes.append(label_dtype_map[entry['DATA_TYPE']])
bytelengths.append(entry['BYTES'])
else:
ncols -= 1
strbytes = map(str, bytelengths)
rowdtype = list(zip(columns, map(''.join, zip(['>'] * ncols, datatypes, strbytes))))
d = np.frombuffer(indata.read(rowbytes * nrows), dtype=rowdtype,
count=nrows)
self.ancillary_data = pd.DataFrame(d, columns=columns,
index=np.arange(nrows))
keys = []
vals = []
for k, v in label.items():
if k in ["ANCILLARY_AND_SUPPLEMENT_DATA", "L2D_RESULT_ARRAY",
"SP_SPECTRUM_QA", "SP_SPECTRUM_REF1", "SP_SPECTRUM_RAD",
"SP_SPECTRUM_REF2", "SP_SPECTRUM_RAW", "SP_SPECTRUM_WAV",
"^ANCILLARY_AND_SUPPLEMENT_DATA", "^SP_SPECTRUM_WAV",
"^SP_SPECTRUM_RAW", "^SP_SPECTRUM_REF2"," ^SP_SPECTRUM_RAD",
"^SP_SPECTRUM_REF1", "^SP_SPECTRUM_QA", "^L2D_RESULT_ARRAY",
"^SP_SPECTRUM_RAD"]:
continue
if isinstance(v, pvl._collections.Units):
k = "{}_{}".format(k, v.units)
v = v.value
keys.append(k)
vals.append(v)
vals = [vals] * len(self.ancillary_data)
new_anc = pd.DataFrame(vals, index=self.ancillary_data.index, columns=keys)
self.ancillary_data = self.ancillary_data.join(new_anc, how='inner')
assert(ncols == len(columns))
keys = []
array_offsets = []
for d in ['WAV', 'RAW', 'REF', 'REF1', 'REF2', 'DAR', 'QA', 'RAD']:
search_key = '^SP_SPECTRUM_{}'.format(d)
result = find_in_dict(label, search_key)
if result:
try:
array_offsets.append(result.value)
except:
array_offsets.append(result[1].value) # 2C V3.0
keys.append('SP_SPECTRUM_{}'.format(d))
offsets = dict(zip(keys, array_offsets))
arrays = {}
for k, offset in offsets.items():
indata.seek(offset - 1)
newk = k.split('_')[-1]
d = find_in_dict(label, k)
unit = d['UNIT']
lines = d['LINES']
scaling_factor = d['SCALING_FACTOR']
arr = np.frombuffer(indata.read(lines * 296*2), dtype='>H').astype(np.float64)
arr = arr.reshape(lines, -1)
# If the data is scaled, apply the scaling factor
if isinstance(scaling_factor, float):
arr *= scaling_factor
arrays[newk] = arr
self.wavelengths = pd.Series(arrays['WAV'][0])
self.spectra = {}
for i in range(nrows):
self.spectra[i] = pd.DataFrame(index=self.wavelengths)
for k in keys:
k = k.split('_')[-1]
if k == 'WAV':
continue
self.spectra[i][k] = arrays[k][i]
if cleaned:
mask = self.spectra[i]['QA'] < qa_threshold
self.spectra[i] = self.spectra[i][mask]
# If the spectra have been cleaned, the wavelength ids also need to be cleaned.
if cleaned:
self.wavelengths = self.wavelengths[mask.values].values
dfs = [v for k, v in self.spectra.items()]
self.spectra = pd.concat(dfs, axis=1, keys=range(nrows))
def _load_label(self, stream):
return pvl.load(stream)
def test_load_all_sample_labels():
for filename in PDS_LABELS:
label = pvl.load(filename)
assert isinstance(label, Label)
def __init__(self, input_data, var_file = None, data_set=None):
"""
Read the .spc file, parse the label, and extract the spectra
Parameters
----------
input_data : string
The PATH to the input .tab file
"""
def expand_column(df, expand_column, columns): # pragma: no cover
array = np.asarray([np.asarray(list(tup[0])) for tup in df[expand_column].as_matrix()], dtype=np.uint8)
new_df = pd.concat([df, pd.DataFrame(array, columns=columns)], axis=1)
del new_df[expand_column]
return new_df
def bolquality2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:3]), bit2bool(bitarr[3:4]))]
types = [('BOLOMETRIC_INERTIA_RATING', '>u1'), ('BOLOMETER_LAMP_ANOMALY', 'bool_')]
arr = np.array(lis, dtype=types)
return arr
def obsquality2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:5]),
bitarr2int(bitarr[5:6]), bitarr2int(bitarr[6:7]),
bitarr2int(bitarr[7:8]), bitarr2int(bitarr[8:9]))]
types = [('HGA_MOTION', '>u1'), ('SOLAR_PANEL_MOTION', '>u1'), ('ALGOR_PATCH', '>u1'),
('IMC_PATCH', '>u1'), ('MOMENTUM_DESATURATION', '>u1'), ('EQUALIZATION_TABLE', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def obsclass2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:3]), bitarr2int(bitarr[3:7]),
bitarr2int(bitarr[7:11]), bitarr2int(bitarr[11:13]),
bitarr2int(bitarr[13:14]), bitarr2int(bitarr[14:16]),
bitarr2int(bitarr[16:]))]
types = [('MISSION_PHASE', '>u1'), ('INTENDED_TARGET', '>u1'), ('TES_SEQUENCE', '>u1'),
('NEON_LAMP_STATUS', '>u1'), ('TIMING_ACCURACY', '>u1'), ('SPARE', '>u1'), ('CLASSIFICATION_VALUE', '>u2')]
arr = np.array(lis, dtype=types)
return arr
def radquality2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:1]), bitarr2int(bitarr[1:2]),
bitarr2int(bitarr[2:3]), bitarr2int(bitarr[3:5]),
bitarr2int(bitarr[5:7]), bitarr2int(bitarr[5:8]),
bitarr2int(bitarr[8:9]))]
types = [('MAJOR_PHASE_INVERSION', '>u1'), ('ALGOR_RISK', '>u1'), ('CALIBRATION_FAILURE', '>u1'),
('CALIBRATION_QUALITY', '>u1'), ('SPECTROMETER_NOISE', '>u1'), ('SPECTRAL_INERTIA_RATING', '>u1'),
('DETECTOR_MASK_PROBLEM', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def atmquality2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:4]))]
types = [('TEMPERATURE_PROFILE_RATING', '>u1'), ('ATMOSPHERIC_OPACITY_RATING', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def expand_column(df, expand_column, columns): # pragma: no cover
array = np.asarray([np.asarray(list(tup[0])) for tup in df[expand_column].as_matrix()], dtype=np.uint8)
new_df = pd.concat([df, pd.DataFrame(array, columns=columns)], axis=1)
del new_df[expand_column]
return new_df
def bolquality2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:3]), bit2bool(bitarr[3:4]))]
types = [('BOLOMETRIC_INERTIA_RATING', '>u1'), ('BOLOMETER_LAMP_ANOMALY', 'bool_')]
arr = np.array(lis, dtype=types)
return arr
def obsquality2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:5]),
bitarr2int(bitarr[5:6]), bitarr2int(bitarr[6:7]),
bitarr2int(bitarr[7:8]), bitarr2int(bitarr[8:9]))]
types = [('HGA_MOTION', '>u1'), ('SOLAR_PANEL_MOTION', '>u1'), ('ALGOR_PATCH', '>u1'),
('IMC_PATCH', '>u1'), ('MOMENTUM_DESATURATION', '>u1'), ('EQUALIZATION_TABLE', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def obsclass2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:3]), bitarr2int(bitarr[3:7]),
bitarr2int(bitarr[7:11]), bitarr2int(bitarr[11:13]),
bitarr2int(bitarr[13:14]), bitarr2int(bitarr[14:16]),
bitarr2int(bitarr[16:]))]
types = [('MISSION_PHASE', '>u1'), ('INTENDED_TARGET', '>u1'), ('TES_SEQUENCE', '>u1'),
('NEON_LAMP_STATUS', '>u1'), ('TIMING_ACCURACY', '>u1'), ('SPARE', '>u1'), ('CLASSIFICATION_VALUE', '>u2')]
arr = np.array(lis, dtype=types)
return arr
def radquality2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:1]), bitarr2int(bitarr[1:2]),
bitarr2int(bitarr[2:3]), bitarr2int(bitarr[3:5]),
bitarr2int(bitarr[5:7]), bitarr2int(bitarr[5:8]),
bitarr2int(bitarr[8:9]))]
types = [('MAJOR_PHASE_INVERSION', '>u1'), ('ALGOR_RISK', '>u1'), ('CALIBRATION_FAILURE', '>u1'),
('CALIBRATION_QUALITY', '>u1'), ('SPECTROMETER_NOISE', '>u1'), ('SPECTRAL_INERTIA_RATING', '>u1'),
('DETECTOR_MASK_PROBLEM', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def atmquality2arr(arr): # pragma: no cover
bitarr = np.unpackbits(np.asarray(arr, dtype=np.uint8))
lis = [(bitarr2int(bitarr[0:2]), bitarr2int(bitarr[2:4]))]
types = [('TEMPERATURE_PROFILE_RATING', '>u1'), ('ATMOSPHERIC_OPACITY_RATING', '>u1')]
arr = np.array(lis, dtype=types)
return arr
def bitarr2int(arr): # pragma: no cover
arr = "".join(str(i) for i in arr)
return np.uint8(int(arr,2))
def bit2bool(bit): # pragma: no cover
return np.bool_(bit)
def expand_bitstrings(df, dataset): # pragma: no cover
if dataset == 'BOL':
quality_columns = ['ti_bol_rating', 'bol_ref_lamp']
df['quality'] = df['quality'].apply(bolquality2arr)
return expand_column(df, 'quality', quality_columns)
elif dataset == 'OBS':
quality_columns = ['hga_motion', 'pnl_motion', 'algor_patch', 'imc_patch',
'momentum', 'equal_tab']
class_columns = ['phase', 'type', 'sequence',
'lamp_status', 'timing', 'spare', 'class_value']
df['quality'] = df['quality'].apply(obsquality2arr)
df['class'] = df['class'].apply(obsclass2arr)
new_df = expand_column(df, 'quality', quality_columns)
new_df = expand_column(new_df, 'class', class_columns)
return new_df
elif dataset == 'RAD':
quality_columns = ['phase_inversion', 'algor_risk', 'calib_fail', 'calib_quality',
'spect_noise', 'ti_spc_rating', 'det_mask_problem']
df['quality'] = df['quality'].apply(radquality2arr)
return expand_column(df, 'quality', quality_columns)
elif dataset == 'ATM':
quality_columns = ['atm_pt_rating', 'atm_opacity_rating']
df['quality'] = df['quality'].apply(atmquality2arr)
return expand_column(df, 'quality', quality_columns)
else:
return df
if isinstance(input_data, pd.DataFrame):
self.dataset = None
if not data_set:
for key in tes_columns.keys():
if len(set(tes_columns[key]).intersection(set(input_data.columns))) > 3 :
self.dataset = key
else:
self.dataset=data_set
self.label = None
self.data = input_data
return
self.label = pvl.load(input_data)
nrecords = self.label['TABLE']['ROWS']
nbytes_per_rec = self.label['RECORD_BYTES']
data_start = self.label['LABEL_RECORDS'] * self.label['RECORD_BYTES']
dataset = self.label['TABLE']['^STRUCTURE'].split('.')[0]
self.dataset = dataset
numpy_dtypes = tes_dtype_map
columns = tes_columns
scaling_factors = tes_scaling_factors
with open(input_data, 'rb') as file:
file.seek(data_start)
buffer = file.read(nrecords*nbytes_per_rec)
array = np.frombuffer(buffer, dtype=numpy_dtypes[dataset.upper()]).byteswap().newbyteorder()
df = pd.DataFrame(data=array, columns=columns[dataset.upper()])
# Read Radiance array if applicable
if dataset.upper() == 'RAD': # pragma: no cover
if not var_file:
filename, file_extension = path.splitext(input_data)
var_file = filename + ".var"
with open(var_file, "rb") as var:
buffer = var.read()
def process_rad(index):
if index is -1:
return None
length = np.frombuffer(buffer[index:index+2], dtype='>u2')[0]
exp = np.frombuffer(buffer[index+2:index+4], dtype='>i2')[0]
scale = 2**(int(exp)-15)
radarr = np.frombuffer(buffer[index+4:index+4+length-2], dtype='>i2') * scale
if np.frombuffer(buffer[index+4+length-2:index+4+length], dtype='>u2')[0] != length:
warnings.warn("Last element did not match the length for file index {} in file {}".format(index, f))
return radarr
df["raw_rad"] = df["raw_rad"].apply(process_rad)
df["cal_rad"] = df["cal_rad"].apply(process_rad)
# Apply scaling factors
for column in scaling_factors[dataset]: # pragma: no cover
def scale(x):
return np.multiply(x, scaling_factors[dataset][column])
df[column] = df[column].apply(scale)
df = expand_bitstrings(df, dataset.upper())
self.data = df
def pvl_lbl(self):
return pvl.load(str(self.path))
def _parse_label(self, stream):
"""Parses the label of an ISIS cube, returns a pvl module"""
return pvl.load(stream)
