def _load(self):
with open(self.lbl_path, 'r') as f:
parser = Parser()
labels = parser.parse(f)
record_bytes = int(labels['RECORD_BYTES'])
nrecords = int(labels['FILE_RECORDS'])
columns = defaultdict(list)
with open(self.data_path, 'rb') as f:
for i in range(nrecords):
for cname, cstart, ctype, citems, _ in ELS.COLUMNS:
# Subtract 1 because they are indexed from 1 in the .FMT
f.seek(i * record_bytes + cstart - 1)
columns[cname].append(
f.read(np.dtype(ctype).itemsize * citems))
for cname, _, ctype, citems, missing in ELS.COLUMNS:
cstr = ''.join(columns[cname])
col = np.fromstring(cstr, dtype=ctype, count=nrecords * citems)
col = np.squeeze(col.reshape((nrecords, citems)))
# Replace missing value with NaN
if missing is not None:
col[col == missing] = np.nan
# Apply post-processing steps to appropriate columns
if cname in ELS.POSTPROCESS:
col = ELS.POSTPROCESS[cname](col)
# Store column as object attribute
setattr(self, cname, col)
# Add iso_data by summing across theta/phi
self.iso_data = np.sum(self.data, axis=(-2, -1))
# Compute DEF, DNF, and PSD
self.def_data = compute_def(self.dim1_e, self.data)
self.dnf_data = compute_dnf(self.dim1_e, self.def_data)
self.psd_data = compute_psd(self.dim1_e, self.def_data)
def get_labels(fname):
parser = Parser()
labels = parser.parse(open_pds(fname))
return labels
# with StringIO.StringIO(pdsContents) as pdsFile:
# with open_pds(pdsContents) as pdsFile:
pdsFile = open_pds(StringIO.StringIO(pdsContents))
# pdsFile = open_pds(pdsContents)
if options.step_through:
sys.stderr.write(
"stepping through files... press enter to continue\n")
raw_input()
if options.verbose:
errorMessage = "Reading input from '%s'\n" % (pdsFilename)
sys.stderr.write(errorMessage)
labels = None
try:
labels = pdsParser.parse(pdsFile)
# labels = pdsParser.parse(open_pds(pdsFile))
except:
if options.ignore_exceptions:
if options.verbose:
errorMessage = "Warn: Caught exception raised while parsing of '%s', ignoring\n" % (
pdsFilename)
sys.stderr.write(errorMessage)
else:
# If not ignoring caught exceptions, re-raise.
raise
finally:
pdsFile.close()
if not labels:
errorMessage = "Error: Could not parse %s: no labels were found\n" % (
# support for fileinput and *StringIO.
# with StringIO.StringIO(pdsContents) as pdsFile:
# with open_pds(pdsContents) as pdsFile:
pdsFile = open_pds(StringIO.StringIO(pdsContents))
# pdsFile = open_pds(pdsContents)
if options.step_through:
sys.stderr.write("stepping through files... press enter to continue\n")
raw_input()
if options.verbose:
errorMessage = "Reading input from '%s'\n" % (pdsFilename)
sys.stderr.write(errorMessage)
labels = None
try:
labels = pdsParser.parse(pdsFile)
# labels = pdsParser.parse(open_pds(pdsFile))
except:
if options.ignore_exceptions:
if options.verbose:
errorMessage = "Warn: Caught exception raised while parsing of '%s', ignoring\n" % (pdsFilename)
sys.stderr.write(errorMessage)
else:
# If not ignoring caught exceptions, re-raise.
raise
finally:
pdsFile.close()
if not labels:
errorMessage = "Error: Could not parse %s: no labels were found\n" % (options.filename)
sys.stderr.write(errorMessage)
def load_image(self, name, rootdir, show=False, rgb=False, ext='.IMG', lblext='.LBL_label'):
imgpath = rootdir + name + ext
lblpath = rootdir + name + lblext
parser = Parser()
labels = parser.parse(open_pds(lblpath))
h = int(labels['IMAGE']['LINES'])
w = int(labels['IMAGE']['LINE_SAMPLES'])
#print "h: %d, w: %d, type: %s" % (h, w, labels['IMAGE']['SAMPLE_TYPE'])
if labels['IMAGE']['SAMPLE_TYPE'] == 'UNSIGNED_INTEGER':
dt = "uint" + labels['IMAGE']['SAMPLE_BITS']
#pprint(labels['IMAGE'])
imgarr = np.fromfile(imgpath, dtype=dt, sep="")
imgarr = imgarr.astype('uint8')
elif labels['IMAGE']['SAMPLE_TYPE'] == 'MSB_UNSIGNED_INTEGER':
#dt = "uint" + labels['IMAGE']['SAMPLE_BITS']
dt = ">u" + str(int(labels['IMAGE']['SAMPLE_BITS'])/8)
#pprint(labels['IMAGE'])
imgarr = np.fromfile(imgpath, dtype=dt, sep="")
#print imgarr.dtype
imgarr = imgarr.byteswap().newbyteorder()
#print imgarr.dtype
#imgarr = np.bitwise_and(4095, imgarr)
imgarr = imgarr.astype('float16')
imgarr = imgarr / 4095 * 255
raw_input("WARNING. THE CODE HAS JUST CHANGED THE DATA VALUES FROM float16 TO uint8.\n"
" THIS WAS INITIALLY DONE FOR VISUALIZATION BUT HAS CAUSED PROBLEMS.\n"
" PLEASE FIX, IF POSSIBLE! NEEDS RE-TYPING OF ALL ARRAYS.\n"
"\n"
"ALSO, THE RADIANCE OFFSET AND SCALING FACTOR IN THE LOG FILES NEED TO BE APPLIED.\n"
"\n"
" ...press enter if you really want to continue. ")
imgarr = imgarr.astype('uint8')
else:
print "Error: unknown sample type: %s" % labels['IMAGE']['SAMPLE_TYPE']
exit()
L = len(imgarr)
B = int(labels['IMAGE']['BANDS'])
#print "%7d %s %s" % (B, name, labels['INSTRUMENT_STATE_PARMS']['FILTER_NAME'])
if B == 1:
#print "One band: %s" % name
#print imgarr.shape
pass
X = L / B
#print "len: %d, h*w: %d, bands: %d, div: %d" % (L, h*w, B, X)
assert (L % B == 0)
#print "max: %d, min: %d, mean: %d" % (max(imgarr), min(imgarr), np.mean(imgarr))
if B == 3:
img = np.zeros((h, w, B), dtype='uint8')
for b in range(B):
img[...,b] = imgarr[b*X:(b+1)*X].reshape([h, w])
elif B == 1:
img = imgarr.reshape([h,w])
img2 = np.zeros((h, w, 1), dtype='uint8')
img2[:,:,0] = img
img = img2
else:
# should never get here
print "Error: Your dimensionality is ridiculous."
print " Why does the image have %d bands?" % B
exit(-1)
if show:
img_show = PIL.Image.fromarray(img)
img_show.save(os.path.join('mastcam_images', '%s_temp.png' % name))
return (img, labels)
def read_mastcam_dir(self, filepath, suffix, unit, feature, extension = '.IMG', lblext='.LBL_label', eye='LR', margin=6):
"""read_mastcam_dir(filepath, extension)
Read in all files with given extension in the folder given by filepath and save to a pickle.
The pickle is called filepath/kepler.extension.pkl
"""
if eye == 'L':
eyez = 'ML'
elif eye == 'R':
eyez = 'MR'
elif eye == 'LR':
eyez = ''
pass
else:
raise ValueError('Eye name %s is not valid! Use L, R, or LR.' % eye)
# GET ALL FILES WITH GIVEN EXTENSION IN FILEPATH
files = sorted(glob.glob(str(filepath) + "*" + eyez + "*" + str(suffix) + "*" + str(extension)))
fileprefixes = sorted(list(set([f.split('/')[-1][0:12] for f in files])))
print fileprefixes
print "found %d files among %d sequences with eye %s and extension %s in %s:" % (len(files), len(fileprefixes), eye, extension, filepath)
assert len(files) > 0
numfiles = len(fileprefixes)
seen = 0
percent = 0.0
printed = [False for foo in range(1000)]
fullimages = {}
segmentation = {}
data = []
self.labels = []
for fileprefix in fileprefixes:
print " " + fileprefix
thissequence = sorted(glob.glob(str(filepath) + fileprefix + "*" + str(suffix) + "*" + str(extension)))
asdfghjkl = 0
parser = Parser()
seqfiltstr = ""
dimlist = []
for w in thissequence:
labels = parser.parse(open_pds(w.replace(extension, lblext)))
filt = labels['INSTRUMENT_STATE_PARMS']['FILTER_NAME'][9]
seqfiltstr += filt
h = int(labels['IMAGE']['LINES'])
w = int(labels['IMAGE']['LINE_SAMPLES'])
dimlist.append([h, w])
#print " %s %s %s" % (filt, h, w)
print "Filter name:", labels['INSTRUMENT_STATE_PARMS']['FILTER_NAME']
#print seqfiltstr
# print dimlist
seqstocombine = []
# Handle cases which appear to be several series of observations
if len(seqfiltstr) % 7 == 0:
for i in range(len(seqfiltstr) / 7):
subseq = thissequence[7*i:7*i+7]
subseqfilt = seqfiltstr[7*i:7*i+7]
if subseqfilt == '0123456':
cont = False
for j in range(7*i, 7*i+7):
if dimlist[7*i] != dimlist[j]:
print "SIZE ERROR"
cont = True
if cont:
continue
seqstocombine.append(subseq)
else:
if seqfiltstr == '00112233445566':
seq1 = [thissequence[2*i] for i in range(len(thissequence)/2)]
seq2 = [thissequence[2*i+1] for i in range(len(thissequence)/2)]
seqstocombine.append(seq1)
seqstocombine.append(seq2)
break
else:
print "Length multiple of 7 but bad sequence"
# Non-7 number of observations
else:
for i in range(len(seqfiltstr)):
subseq = thissequence[i:i+7]
subseqfilt = seqfiltstr[i:i+7]
if subseqfilt == '0123456':
cont = False
for j in range(i, i+7):
if dimlist[i] != dimlist[j]:
print "SIZE ERROR"
cont = True
if cont: continue
seqstocombine.append(subseq)
# No actual multispectral images exist, so use all RGB (sol 388)
if len(seqstocombine) == 0 and 'sol388' in self.archive:
seqstocombine = [[f] for f in thissequence]
# Now, download each sequence with this prefix
for subseq in seqstocombine:
qwertyuiop = 0
bigimage = None
err = False
# Get each image within sequence
for filename in subseq:
namestem = filename.split('.')[0].split('/')[-1]
try:
(image, lbls) = self.load_image(namestem, filepath, ext=extension, lblext=lblext)
except ValueError as e:
#print "An error happened while processing %s" % filename
err = True
break
(h, w, b) = image.shape
if b == 3:
self.rgbdict[fileprefix + str(asdfghjkl)] = namestem
fullimages[fileprefix + str(asdfghjkl)] = image
#print "Stored %s to rgbdict" % (fileprefix + str(asdfghjkl))
if bigimage == None and 'sol388' not in filepath:
bigimage = np.zeros([h, w, 9], dtype='uint8')
elif bigimage == None:
bigimage = np.zeros([h, w, b], dtype='uint8')
bigimage[:,:,qwertyuiop:qwertyuiop+b] = image
qwertyuiop += b
# Reorder images based on camera so filters are ordered
if eye in ['L', 'R']:
bi = np.zeros([h, w, 9], dtype='uint8')
if eye == 'L':
bi[:, :, 0] = bigimage[:, :, 0]
bi[:, :, 1] = bigimage[:, :, 1]
bi[:, :, 2] = bigimage[:, :, 2]
bi[:, :, 3] = bigimage[:, :, 4]
bi[:, :, 4] = bigimage[:, :, 3]
bi[:, :, 5] = bigimage[:, :, 6]
bi[:, :, 6] = bigimage[:, :, 5]
bi[:, :, 7] = bigimage[:, :, 7]
bi[:, :, 8] = bigimage[:, :, 8]
elif eye == 'R':
bi[:, :, 0] = bigimage[:, :, 2]
bi[:, :, 1] = bigimage[:, :, 1]
bi[:, :, 2] = bigimage[:, :, 0]
bi[:, :, 3] = bigimage[:, :, 4]
bi[:, :, 4] = bigimage[:, :, 3]
bi[:, :, 5] = bigimage[:, :, 5]
bi[:, :, 6] = bigimage[:, :, 6]
bi[:, :, 7] = bigimage[:, :, 7]
bi[:, :, 8] = bigimage[:, :, 8]
bigimage = bi
if err:
print " ...didn't load sequence. There was an error."
continue
print " ...loaded one sequence:", (fileprefix + str(asdfghjkl))
if 'sol388' not in self.archive:
name = fileprefix + str(asdfghjkl) + '_' + unit + '_' + feature
else:
name = namestem + '_' + unit + '_' + feature
(segments, segmentlabels) = self.segment_image(bigimage, unit=unit)
segmentation[fileprefix + str(asdfghjkl)] = segments[0][1]
for i in range(len(segments)):
data += [[float(x) for x in self.process_image(segments[i], name + segmentlabels[i], feature=feature)]]
asdfghjkl += 1
###########################################
seen += 1
# output read-in progress
if percent < 100:
if (round((seen / float(numfiles)) * 100, 1) >= percent) and (printed[int(percent * 10)] == False):
#print "...%3.1f%%..." % percent
printed[int(percent * 10)] == True
percent = round(((seen / float(numfiles)) * 100), 1) + 1
print "...100%..."
print "Transposing data..."
data = np.array(data).T
self.xvals.sort()
# Output the pickle
print "Writing pickle to " + self.archive + " ..."
outf = open(self.archive, 'w')
pickle.dump((data, fullimages, segmentation, self.labels, self.xlabel, self.ylabel, self.xvals, self.rgbdict, self.lblext, self.initdata, self.initfilename), outf)
outf.close()
print "Wrote pickle to " + self.archive
if sample == missingconst:
#print '.',
pass
else:
pass
#if value>minval and value<maxval: debug( 'ok' )
#else: debug( 'out of range' )
#print value ,
#debug( value )
p = Parser()
filename = '/tmp/x.img'
file = open_pds(filename)
label = p.parse( file )
if '--label' in string.join(sys.argv):
pprint(label)
sys.exit(0)
if '--debug' in string.join(sys.argv): debugon=True
status = verify_label(label)
if status!='ok':
debug('error:',status)
else:
debug('label verified')
print_info(label)
process_samples( filename, label )
def load_image(self, name, rootdir, show=False, rgb=False, ext='.IMG', lblext='.LBL_label'):
imgpath = rootdir + name + ext
lblpath = rootdir + name + lblext
parser = Parser()
labels = parser.parse(open_pds(lblpath))
h = int(labels['IMAGE']['LINES'])
w = int(labels['IMAGE']['LINE_SAMPLES'])
#print "h: %d, w: %d, type: %s" % (h, w, labels['IMAGE']['SAMPLE_TYPE'])
if labels['IMAGE']['SAMPLE_TYPE'] == 'UNSIGNED_INTEGER':
dt = "uint" + labels['IMAGE']['SAMPLE_BITS']
#pprint(labels['IMAGE'])
imgarr = np.fromfile(imgpath, dtype=dt, sep="")
imgarr = imgarr.astype('uint8')
elif labels['IMAGE']['SAMPLE_TYPE'] == 'MSB_UNSIGNED_INTEGER':
#dt = "uint" + labels['IMAGE']['SAMPLE_BITS']
dt = ">u" + str(int(labels['IMAGE']['SAMPLE_BITS'])/8)
#pprint(labels['IMAGE'])
imgarr = np.fromfile(imgpath, dtype=dt, sep="")
#print imgarr.dtype
imgarr = imgarr.byteswap().newbyteorder()
#print imgarr.dtype
#imgarr = np.bitwise_and(4095, imgarr)
imgarr = imgarr.astype('float16')
imgarr = imgarr / 4095 * 255
raw_input("WARNING. THE CODE HAS JUST CHANGED THE DATA VALUES FROM float16 TO uint8.\n"
" THIS WAS INITIALLY DONE FOR VISUALIZATION BUT HAS CAUSED PROBLEMS.\n"
" PLEASE FIX, IF POSSIBLE! NEEDS RE-TYPING OF ALL ARRAYS.\n"
"\n"
"ALSO, THE RADIANCE OFFSET AND SCALING FACTOR IN THE LOG FILES NEED TO BE APPLIED.\n"
"\n"
" ...press enter if you really want to continue. ")
imgarr = imgarr.astype('uint8')
else:
print "Error: unknown sample type: %s" % labels['IMAGE']['SAMPLE_TYPE']
exit()
L = len(imgarr)
B = int(labels['IMAGE']['BANDS'])
#print "%7d %s %s" % (B, name, labels['INSTRUMENT_STATE_PARMS']['FILTER_NAME'])
if B == 1:
#print "One band: %s" % name
#print imgarr.shape
pass
X = L / B
#print "len: %d, h*w: %d, bands: %d, div: %d" % (L, h*w, B, X)
assert (L % B == 0)
#print "max: %d, min: %d, mean: %d" % (max(imgarr), min(imgarr), np.mean(imgarr))
if B == 3:
img = np.zeros((h, w, B), dtype='uint8')
for b in range(B):
img[...,b] = imgarr[b*X:(b+1)*X].reshape([h, w])
elif B == 1:
img = imgarr.reshape([h,w])
img2 = np.zeros((h, w, 1), dtype='uint8')
img2[:,:,0] = img
img = img2
else:
# should never get here
print "Error: Your dimensionality is ridiculous."
print " Why does the image have %d bands?" % B
exit(-1)
if show:
img_show = PIL.Image.fromarray(img)
img_show.save(os.path.join('mastcam_images', '%s_temp.png' % name))
return (img, labels)
def read_mastcam_dir(self, filepath, suffix, unit, feature, extension = '.IMG', lblext='.LBL_label', eye='LR', margin=6):
"""read_mastcam_dir(filepath, extension)
Read in all files with given extension in the folder given by filepath and save to a pickle.
The pickle is called filepath/kepler.extension.pkl
"""
if eye == 'L':
eyez = 'ML'
elif eye == 'R':
eyez = 'MR'
elif eye == 'LR':
eyez = ''
pass
else:
raise ValueError('Eye name %s is not valid! Use L, R, or LR.' % eye)
# GET ALL FILES WITH GIVEN EXTENSION IN FILEPATH
files = sorted(glob.glob(str(filepath) + "*" + eyez + "*" + str(suffix) + "*" + str(extension)))
fileprefixes = sorted(list(set([f.split('/')[-1][0:12] for f in files])))
print fileprefixes
print "found %d files among %d sequences with eye %s and extension %s in %s:" % (len(files), len(fileprefixes), eye, extension, filepath)
assert len(files) > 0
numfiles = len(fileprefixes)
seen = 0
percent = 0.0
printed = [False for foo in range(1000)]
fullimages = {}
segmentation = {}
data = []
self.labels = []
for fileprefix in fileprefixes:
print " " + fileprefix
thissequence = sorted(glob.glob(str(filepath) + fileprefix + "*" + str(suffix) + "*" + str(extension)))
asdfghjkl = 0
parser = Parser()
seqfiltstr = ""
dimlist = []
for w in thissequence:
labels = parser.parse(open_pds(w.replace(extension, lblext)))
filt = labels['INSTRUMENT_STATE_PARMS']['FILTER_NAME'][9]
seqfiltstr += filt
h = int(labels['IMAGE']['LINES'])
w = int(labels['IMAGE']['LINE_SAMPLES'])
dimlist.append([h, w])
#print " %s %s %s" % (filt, h, w)
print "Filter name:", labels['INSTRUMENT_STATE_PARMS']['FILTER_NAME']
#print seqfiltstr
# print dimlist
seqstocombine = []
# Handle cases which appear to be several series of observations
if len(seqfiltstr) % 7 == 0:
for i in range(len(seqfiltstr) / 7):
subseq = thissequence[7*i:7*i+7]
subseqfilt = seqfiltstr[7*i:7*i+7]
if subseqfilt == '0123456':
cont = False
for j in range(7*i, 7*i+7):
if dimlist[7*i] != dimlist[j]:
print "SIZE ERROR"
cont = True
if cont:
continue
seqstocombine.append(subseq)
else:
if seqfiltstr == '00112233445566':
seq1 = [thissequence[2*i] for i in range(len(thissequence)/2)]
seq2 = [thissequence[2*i+1] for i in range(len(thissequence)/2)]
seqstocombine.append(seq1)
seqstocombine.append(seq2)
break
else:
print "Length multiple of 7 but bad sequence"
# Non-7 number of observations
else:
for i in range(len(seqfiltstr)):
subseq = thissequence[i:i+7]
subseqfilt = seqfiltstr[i:i+7]
if subseqfilt == '0123456':
cont = False
for j in range(i, i+7):
if dimlist[i] != dimlist[j]:
print "SIZE ERROR"
cont = True
if cont: continue
seqstocombine.append(subseq)
# No actual multispectral images exist, so use all RGB (sol 388)
if len(seqstocombine) == 0 and 'sol388' in self.archive:
seqstocombine = [[f] for f in thissequence]
# Now, download each sequence with this prefix
for subseq in seqstocombine:
qwertyuiop = 0
bigimage = None
err = False
# Get each image within sequence
for filename in subseq:
namestem = filename.split('.')[0].split('/')[-1]
try:
(image, lbls) = self.load_image(namestem, filepath, ext=extension, lblext=lblext)
except ValueError as e:
#print "An error happened while processing %s" % filename
err = True
break
(h, w, b) = image.shape
if b == 3:
self.rgbdict[fileprefix + str(asdfghjkl)] = namestem
fullimages[fileprefix + str(asdfghjkl)] = image
#print "Stored %s to rgbdict" % (fileprefix + str(asdfghjkl))
if bigimage == None and 'sol388' not in filepath:
bigimage = np.zeros([h, w, 9], dtype='uint8')
elif bigimage == None:
bigimage = np.zeros([h, w, b], dtype='uint8')
bigimage[:,:,qwertyuiop:qwertyuiop+b] = image
qwertyuiop += b
# Reorder images based on camera so filters are ordered
if eye in ['L', 'R']:
bi = np.zeros([h, w, 9], dtype='uint8')
if eye == 'L':
bi[:, :, 0] = bigimage[:, :, 0]
bi[:, :, 1] = bigimage[:, :, 1]
bi[:, :, 2] = bigimage[:, :, 2]
bi[:, :, 3] = bigimage[:, :, 4]
bi[:, :, 4] = bigimage[:, :, 3]
bi[:, :, 5] = bigimage[:, :, 6]
bi[:, :, 6] = bigimage[:, :, 5]
bi[:, :, 7] = bigimage[:, :, 7]
bi[:, :, 8] = bigimage[:, :, 8]
elif eye == 'R':
bi[:, :, 0] = bigimage[:, :, 2]
bi[:, :, 1] = bigimage[:, :, 1]
bi[:, :, 2] = bigimage[:, :, 0]
bi[:, :, 3] = bigimage[:, :, 4]
bi[:, :, 4] = bigimage[:, :, 3]
bi[:, :, 5] = bigimage[:, :, 5]
bi[:, :, 6] = bigimage[:, :, 6]
bi[:, :, 7] = bigimage[:, :, 7]
bi[:, :, 8] = bigimage[:, :, 8]
bigimage = bi
if err:
print " ...didn't load sequence. There was an error."
continue
print " ...loaded one sequence:", (fileprefix + str(asdfghjkl))
if 'sol388' not in self.archive:
name = fileprefix + str(asdfghjkl) + '_' + unit + '_' + feature
else:
name = namestem + '_' + unit + '_' + feature
(segments, segmentlabels) = self.segment_image(bigimage, unit=unit)
segmentation[fileprefix + str(asdfghjkl)] = segments[0][1]
for i in range(len(segments)):
data += [[float(x) for x in self.process_image(segments[i], name + segmentlabels[i], feature=feature)]]
asdfghjkl += 1
###########################################
seen += 1
# output read-in progress
if percent < 100:
if (round((seen / float(numfiles)) * 100, 1) >= percent) and (printed[int(percent * 10)] == False):
#print "...%3.1f%%..." % percent
printed[int(percent * 10)] == True
percent = round(((seen / float(numfiles)) * 100), 1) + 1
print "...100%..."
print "Transposing data..."
data = np.array(data).T
self.xvals.sort()
# Output the pickle
print "Writing pickle to " + self.archive + " ..."
outf = open(self.archive, 'w')
pickle.dump((data, fullimages, segmentation, self.labels, self.xlabel, self.ylabel, self.xvals, self.rgbdict, self.lblext, self.initdata, self.initfilename), outf)
outf.close()
print "Wrote pickle to " + self.archive