def prep_rows(prefixes, backplane, blocker, column_descs,
planet, moon=None, moon_length=0,
tiles=[], tiling_min=100, ignore_shadows=False,
start_index=1, allow_zero_rows=False):
"""Generates the metadata and returns a list of lists of strings. The inner
list contains string representations for each column in one row of the
output file. These will be concatenated with commas between them and written
to the file. The outer list contains one list for each output row. The
number of output rows can be zero or more.
The tiles argument supports detailed listings where a geometric region is
broken down into separate subregions. If the tiles argument is empty (which
is the default), then this routine writes a summary file.
If the tiles argument is not empty, then the routine writes a detailed file,
which generally contains one record for each non-empty subregion. The tiles
argument must be a list of boolean backplane keys, each equal to True for
the pixels within the subregion. An additional column is added before the
geometry columns, containing the index value of the associated tile.
The first backplane in the list is treated differently. It should evaluate
to an area roughly equal to the union of all the other backplanes. It is
used as an overlay to all subsequent tiles.
In a summmary listing, this routine writes one record per call, even if all
values are null. In a detailed listing, only records associated with
non-empty regions of the meshgrid are written.
Input:
prefixes a list of the strings to appear at the beginning of the
line, up to and including the ring observation ID. Each
individual string should already be enclosed in quotes.
backplane backplane for the observation.
blocker the name of one moon that may be able to block or shadow
other bodies.
column_descs a list of column descriptions.
planet name of planet, uppercase, e.g., "SATURN".
moon optionally, the moon name to write into the record.
moon_length the character width of a column to contain moon names.
If zero (which is the default), then no moon name is
written into the record.
tiles an optional list of boolean backplane keys, used to
support the generation of detailed tabulations instead
of summary tabulations. See details above.
tiling_min the lower limit on the number of meshgrid points in a
region before that region is subdivided into tiles.
ignore_shadows True to ignore any mask constraints applicable to
shadowing or to the sunlit faces of surfaces.
start_index index to use for first subregion. Default 1.
allow_zero_rows True to allow the function to return no rows. If False,
a row filled with null values will be returned if
necessary.
"""
# Handle option for multiple tile sets
if type(tiles) == tuple:
rows = []
local_index = start_index
for tile in tiles:
new_rows = prep_rows(prefixes, backplane, blocker, column_descs,
planet, moon, moon_length,
tile, tiling_min, ignore_shadows,
local_index, allow_zero_rows=True)
rows += new_rows
local_index += len(tile) - 1
if rows or allow_zero_rows:
return rows
return prep_rows(prefixes, backplane, blocker, column_descs,
planet, moon, moon_length,
[], tiling_min, ignore_shadows,
start_index, allow_zero_rows=False)
# Handle a single set of tiles
if tiles:
global_area = backplane.evaluate(tiles[0]).vals
subregion_masks = [np.logical_not(global_area)]
if global_area.sum() < tiling_min:
tiles = []
else:
for tile in tiles[1:]:
mask = backplane.evaluate(tile).vals & global_area
subregion_masks.append(np.logical_not(mask))
else:
subregion_masks = []
# Initialize the list of rows
rows = []
# Create all the needed pixel masks
excluded_mask_dict = {}
for column_desc in column_descs:
event_key = column_desc[0]
mask_desc = column_desc[1]
target = event_key[1]
key = (target,) + mask_desc
if key in excluded_mask_dict: continue
excluded_mask_dict[key] = \
construct_excluded_mask(backplane, target, planet, mask_desc,
blocker, ignore_shadows)
# Interpret the subregion list
if tiles:
indices = range(1,len(tiles))
else:
indices = [0]
# For each subregion...
for indx in indices:
# Skip a subregion if it will be empty
if indx != 0 and np.all(subregion_masks[indx]): continue
# Initialize the list of columns
prefix_columns = list(prefixes) # make a copy
# Append the moon name if necessary
if moon_length > 0:
lmoon = len(moon)
if lmoon > moon_length:
entry = '"' + moon[:moon_length] + '"'
else:
entry = '"' + moon + (moon_length - lmoon) * ' ' + '"'
prefix_columns.append(entry)
# Insert the subregion index
if subregion_masks:
prefix_columns.append('%2d' % (indx + start_index - 1))
# Append the backplane columns
data_columns = []
nothing_found = True
# For each column...
for column_desc in column_descs:
event_key = column_desc[0]
mask_desc = column_desc[1]
# Fill in the backplane array
if event_key[1] == NULL:
values = oops.Scalar(0., True)
else:
values = backplane.evaluate(event_key)
# Make a shallow copy and apply the new masks
target = event_key[1]
excluded = excluded_mask_dict[(target,) + mask_desc]
values = values.mask_where(excluded)
if len(subregion_masks) > 1:
values = values.mask_where(subregion_masks[indx])
elif len(subregion_masks) == 1:
values = values.mask_where(subregion_masks[0])
if not np.all(values.mask):
nothing_found = False
# Write the column using the specified format
if len(column_desc) > 2:
format = ALT_FORMAT_DICT[(event_key[0], column_desc[2])]
else:
format = FORMAT_DICT[event_key[0]]
data_columns.append(formatted_column(values, format))
# Save the row if it was completed
if len(data_columns) < len(column_descs): continue # hopeless error
if nothing_found and (indx > 0 or allow_zero_rows): continue
rows.append(prefix_columns + data_columns)
# Return something if we can
if rows or allow_zero_rows:
return rows
return prep_rows(prefixes, backplane, blocker, column_descs,
planet, moon, moon_length,
[], 0, ignore_shadows, start_index,
allow_zero_rows=False)
def prep_rows(prefixes,
backplane,
blocker,
column_descs,
planet,
moon=None,
moon_length=0,
count_length=0,
tiles=[],
tiling_min=100,
ignore_shadows=False):
"""Generates the metadata and returns a list of lists of strings. The inner
list contains string representations for each column in one row of the
output file. These will be concatenated with commas between them and written
to the file. The outer list contains one list for each output row. The
number of output rows can be zero or more.
The tiles argument supports detailed listings where a geometric region is
broken down into separate subregions. If the tiles argument is empty (which
is the default), then this routine writes a summary file.
If the tiles argument is not empty, then the routine writes a detailed file,
which generally contains one record for each non-empty subregion. The tiles
argument must be a list of boolean backplane keys, each equal to True for
the pixels within the subregion. An additional column is added before the
geometry columns, containing the index value of the associated tile.
The first backplane in the list is treated differently. It should evaluate
to an area roughly equal to the union of all the other backplanes. It is
used as an overlay to all subsequent tiles.
In a summmary listing, this routine writes one record per call, even if all
values are null. In a detailed listing, only records associated with
non-empty regions of the meshgrid are written.
Input:
prefixes a list of the strings to appear at the beginning of the
line, up to and including the ring observation ID. Each
individual string should already be enclosed in quotes.
backplane backplane for the observation.
blocker the name of one moon that may be able to block or shadow
other bodies.
column_descs a list of column descriptions.
planet name of planet, uppercase, e.g., "SATURN".
moon optionally, the moon name to write into the record.
moon_length the character width of a column to contain moon names.
If zero (which is the default), then no moon name is
written into the record.
count_length the character width of a column to contain the number
if samples obtained on the body or within a tile; zero
(which is the default) to suppress this column.
tiles an optional list of boolean backplane keys, used to
support the generation of detailed tabulations instead
of summary tabulations. See details above.
tiling_min the lower limit on the number of meshgrid points in a
region before that region is subdivided into tiles.
ignore_shadows True to ignore any mask constraints applicable to
shadowing or to the sunlit faces of surfaces.
"""
# Handle option for tiles
if tiles:
subregion_masks = [None] # let zone indexing start at 1
# Handle a 1-D set of regions
if len(tiles) == 1:
indices = range(1, len(tiles[0]) + 1)
for tile in tiles[0]:
mask = backplane.evaluate(tile).vals
subregion_masks.append(np.logical_not(mask))
# Handle a 2-D set of regions
else:
indices = range(1, len(tiles[0]) * len(tiles[1]) + 1)
for tile1 in tiles[0]:
mask1 = backplane.evaluate(tile1).vals
for tile2 in tiles[1]:
mask2 = backplane.evaluate(tile2).vals & mask1
subregion_masks.append(np.logical_not(mask2))
# Check size of tile
pixels = 0
for mask in subregion_masks[1:]:
pixels += mask.size - np.count_nonzero(mask)
if pixels < tiling_min:
return []
# Without tiling, the subregion index is zero
else:
indices = [0]
subregion_masks = []
# Create all the needed pixel masks
excluded_mask_dict = {}
for column_desc in column_descs:
event_key = column_desc[0]
mask_desc = column_desc[1]
target = event_key[1]
key = (target, ) + mask_desc
if key in excluded_mask_dict: continue
excluded_mask = construct_excluded_mask(backplane, target, planet,
mask_desc, blocker,
ignore_shadows)
excluded_mask_dict[key] = excluded_mask
# Initialize the list of rows
rows = []
# For each subregion...
for indx in indices:
# Skip a tile if it contains no pixels
if indx != 0 and np.all(subregion_masks[indx]): continue
# Initialize the list of columns
prefix_columns = list(prefixes) # make a copy
# Append the moon name if necessary
if moon_length > 0:
lmoon = len(moon)
if lmoon > moon_length:
entry = '"' + moon[:moon_length] + '"'
else:
entry = '"' + moon + (moon_length - lmoon) * ' ' + '"'
prefix_columns.append(entry)
# Insert the subregion zone if necessary
if tiles:
prefix_columns.append('%2d' % indx)
# Prepare the backplane columns
data_columns = []
# For each column...
pixel_count = 0
for column_desc in column_descs:
event_key = column_desc[0]
mask_desc = column_desc[1]
# Fill in the backplane array
if event_key[1] == NULL:
values = oops.Scalar(0., True)
else:
values = backplane.evaluate(event_key)
# Make a shallow copy and apply the new masks
target = event_key[1]
excluded = excluded_mask_dict[(target, ) + mask_desc]
values = values.mask_where(excluded)
if tiles:
values = values.mask_where(subregion_masks[indx])
if values.size > 1:
pixel_count = max(pixel_count, values.unmasked())
# Write the column using the specified format
if len(column_desc) > 2:
format = ALT_FORMAT_DICT[(event_key[0], column_desc[2])]
else:
format = FORMAT_DICT[event_key[0]]
data_columns.append(formatted_column(values, format))
# Save the row if it was completed
if len(data_columns) < len(column_descs): continue # hopeless error
if indx > 0 and pixel_count == 0: continue
# Generate the sample count column if necessary
if count_length:
count_str = str(pixel_count)
lcount = len(count_str)
if count_length > lcount:
count_str = (count_length - lcount) * ' ' + count_str
elif count_length < lcount:
count_str = count_length * '9' # truncate so it fits
count_columns = [count_str]
else:
count_columns = []
rows.append(prefix_columns + count_columns + data_columns)
return rows
def process_file(root, name, index, files, selection="S"):
# Don't abort if cspice throws a runtime error
try:
(volume_id, file_spec) = volume_and_filespec(root, name)
roid = ring_observation_id(name)
logstr = "%s %4d/%4d %s" % (volume_id, index, files, roid)
print logstr,
terminated_print = False
i20 = name.index("20")
year = int(name[i20:i20 + 4])
if year < 2004:
print "skipped"
return
prefixes = [
'"' + volume_id + '"', '"' + file_spec + '"', '"' + roid + '"'
]
# Create the observation object
obs = uvis.from_file(os.path.join(root, name), enclose=True)
# Get the target
target = obs.dict["TARGET_NAME"].upper()
if target in TRANSLATIONS.keys():
target = TRANSLATIONS[target]
print " " + target
terminated_print = True
# Create the backplane
cad = obs.cadence
if cad.steps <= MAX_TIMESTEPS:
steps = cad.steps
tstride = cad.tstride
else:
steps = MAX_TIMESTEPS
tstride = (cad.time[1] - cad.time[0]) / steps
time_vals = cad.tstart + tstride * np.arange(0.5, steps)
if len(obs.shape) > 1:
time = oops.Scalar(time_vals[:, np.newaxis])
else:
time = oops.Scalar(time_vals)
backplane = oops.Backplane(obs, time=time)
# Catch SPICE NOFRAMECONNECT exceptions before going any further
ignore = backplane.right_ascension()
# Define a list of moon names and the blocker moon
if target in SYSTEM_NAMES and target != PLANET:
blocker = target
moon_names = [target]
else:
blocker = None
moon_names = []
# Write the summary files for the rings and for the planet
if "S" in selection:
meta.write_record(prefixes,
backplane,
blocker,
ring_summary,
RING_SUMMARY_COLUMNS,
PLANET,
ignore_shadows=True)
meta.write_record(prefixes,
backplane,
blocker,
planet_summary,
PLANET_SUMMARY_COLUMNS,
PLANET,
moon=PLANET,
moon_length=NAME_LENGTH,
ignore_shadows=True)
# Write the detailed files for the rings and for the planet
if "D" in selection:
meta.write_record(prefixes,
backplane,
blocker,
ring_detailed,
RING_DETAILED_COLUMNS,
PLANET,
tiles=RING_TILES,
ignore_shadows=True)
meta.write_record(prefixes,
backplane,
blocker,
planet_detailed,
PLANET_DETAILED_COLUMNS,
PLANET,
moon=PLANET,
moon_length=NAME_LENGTH,
tiles=PLANET_TILES,
ignore_shadows=True)
# Write the moon files
for name in moon_names:
if "S" in selection:
meta.write_record(prefixes,
backplane,
blocker,
moon_summary,
MOON_SUMMARY_DICT[name],
PLANET,
moon=name,
moon_length=NAME_LENGTH,
ignore_shadows=True)
if "D" in selection:
meta.write_record(prefixes,
backplane,
blocker,
moon_detailed,
MOON_DETAILED_DICT[name],
PLANET,
moon=name,
moon_length=NAME_LENGTH,
tiles=MOON_TILE_DICT[name],
ignore_shadows=True)
# A RuntimeError is probably caused by missing spice data. There is
# probably nothing we can do.
except RuntimeError as e:
if "NOFRAMECONNECT" in str(e):
print "**** SPICE(NOFRAMECONNECT)"
noframe_file.write(PREFIX + "\n")
else:
if not terminated_print: print
print e
error_file.write(40 * "*" + "\n" + logstr + "\n")
error_file.write(str(e))
error_file.write("\n\n")
# Other kinds of errors are genuine bugs. For now, we just log the
# problem, and jump over the image; we can deal with it later.
except (AssertionError, AttributeError, IndexError, KeyError, LookupError,
TypeError, ValueError, ZeroDivisionError,
pyparsing.ParseException):
if not terminated_print: print
traceback.print_exc()
error_file.write(40 * "*" + "\n" + logstr + "\n")
error_file.write(traceback.format_exc())
error_file.write("\n\n")
def prep_rows(prefixes,
backplane,
blocker,
column_descs,
planet,
moon=None,
moon_length=0,
tiles=[],
tiling_min=100,
ignore_shadows=False):
"""Generates the metadata and returns a list of lists of strings. The inner
list contains string representations for each column in one row of the
output file. These will be concatenated with commas between them and written
to the file. The outer list contains one list for each output row. The
number of output rows can be zero or more.
The tiles argument supports detailed listings where a geometric region is
broken down into separate subregions. If the tiles argument is empty (which
is the default), then this routine writes a summary file.
If the tiles argument is not empty, then the routine writes a detailed file,
which generally contains one record for each non-empty subregion. The tiles
argument must be a list of boolean backplane keys, each equal to True for
the pixels within the subregion. An additional column is added before the
geometry columns, containing the index value of the associated tile.
The first backplane in the list is treated differently. It should evaluate
to an area roughly equal to the union of all the other backplanes. It is
used to ensure that tiling is suppressed when the region to be tiled is too
small. If the number of meshgrid samples that are equal to True in this
backplane is smaller than the limit specified by argument tiling_min, then
no detailed record is written.
In a summmary listing, this routine writes one record per call, even if all
values are null. In a detailed listing, only records associated with
non-empty regions of the meshgrid are written.
Input:
prefixes a list of the strings to appear at the beginning of the
line, up to and including the ring observation ID. Each
individual string should already be enclosed in quotes.
backplane backplane for the observation.
blocker the name of one moon that may be able to block or shadow
other bodies.
column_descs a list of column descriptions.
planet name of planet, uppercase, e.g., "SATURN".
moon optionally, the moon name to write into the record.
moon_length the character width of a column to contain moon names.
If zero (which is the default), then no moon name is
written into the record.
tiles an optional list of boolean backplane keys, used to
support the generation of detailed tabulations instead
of summary tabulations. See details above.
tiling_min the lower limit on the number of meshgrid points in a
region before that region is subdivided into tiles.
ignore_shadows True to ignore any mask constraints applicable to
shadowing or to the sunlit faces of surfaces.
"""
# Initialize the list of rows
rows = []
column_count = len(prefixes) + len(column_descs)
# Decide whether to write a detailed record
if tiles != []:
test_area = backplane.evaluate(tiles[0])
if test_area.sum() < tiling_min: return rows
# Create all the needed pixel masks
mask_dict = {}
for column_desc in column_descs:
event_key = column_desc[0]
mask_desc = column_desc[1]
target = event_key[1]
construct_mask(mask_dict, backplane, target, planet, mask_desc,
blocker, ignore_shadows)
# Interpret the subregion list
if tiles == []:
append_digits = 0
indices = [0]
else:
indices = range(1, len(tiles))
if indices[-1] < 10:
append_digits = 1
else:
append_digits = 2
# For each subregion...
for index in indices:
# Initialize the list of columns
columns = []
for item in prefixes:
columns.append(item)
if tiles == []:
subregion_mask = False
else:
true_where_included = backplane.evaluate(tiles[index])
subregion_mask = (~true_where_included).vals
# Skip this output row if it will be empty
if append_digits > 0:
if np.all(subregion_mask): continue
all_masks_are_empty = True
for mask in mask_dict.values():
if not np.all(mask & subregion_mask):
all_masks_are_empty = False
continue
if all_masks_are_empty: continue
# Append the moon name if necessary
if moon_length > 0:
column_count += 1
lmoon = len(moon)
if lmoon > moon_length:
columns.append('"' + moon[:moon_length] + '"')
else:
columns.append('"' + moon + (moon_length - lmoon) * ' ' + '"')
# Append the detailed record number if necessary
if append_digits == 1:
column_count += 1
columns.append("%1d" % index)
elif append_digits == 2:
column_count += 1
columns.append("%2d" % index)
try:
# For each column...
for column_desc in column_descs:
event_key = column_desc[0]
mask_desc = column_desc[1]
# Fill in the backplane array
if event_key[1] == NULL:
values = oops.Scalar(0., True)
else:
values = backplane.evaluate(event_key)
# Make a shallow copy and apply the new masks
target = event_key[1]
mask = mask_dict[(target, ) + mask_desc]
values = oops.Scalar(values.vals,
mask | values.mask | subregion_mask)
# Write the column using the specified format
if len(column_desc) > 2:
format = ALT_FORMAT_DICT[(event_key[0], column_desc[2])]
else:
format = FORMAT_DICT[event_key[0]]
columns.append(formatted_column(values, format))
# Save the row if it was completed
finally:
if len(columns) == column_count:
rows.append(columns)
return rows