def _transfer_metadata(self):
info = self.l1b.info
# Processing System info
info.set_attribute("pysiral_version", PYSIRAL_VERSION)
# General CryoSat-2 metadata
info.set_attribute("mission", self._mission)
info.set_attribute("mission_data_version", self.cs2l1b.baseline)
info.set_attribute("orbit", self.cs2l1b.sph.abs_orbit_start)
info.set_attribute("cycle", self.cs2l1b.mph.cycle)
mission_data_source = filename_from_path(self.cs2l1b.filename)
info.set_attribute("mission_data_source", mission_data_source)
# Get the product timeliness from the processor stage code
# `proc_stage` in the L1b main product header uses the following
# conventions: N=Near-Real Time, T=Test, O=Off Line (Systematic),
# R=Reprocessing, L=Long Term Archive
#
# We do want to use the following timelines codes adopted from the
# S3 notation: NRT: Near-real time, STC: short-time critical
# NTC: Non-Time Critical, REP: Reprocessed
#
# We therefore define a dictionary and use NTC as default value
timeliness_dct = {"N": "NRT", "O": "NTC", "R": "REP", "L": "REP"}
proc_stage = self.cs2l1b.mph.get_by_fieldname("proc_stage")[0]
info.set_attribute("timeliness", timeliness_dct.get(proc_stage, "NTC"))
# Time-Orbit Metadata
start_time = parse_datetime_str(self.cs2l1b.sph.start_record_tai_time)
stop_time = parse_datetime_str(self.cs2l1b.sph.stop_record_tai_time)
info.set_attribute("start_time", start_time)
info.set_attribute("stop_time", stop_time)
def _read_l1b_file(self, l1b_file):
""" Read a L1b data file (l1bdata netCDF) """
filename = filename_from_path(l1b_file)
self.log.info("- Parsing l1bdata file: %s" % filename)
l1b = L1bdataNCFile(l1b_file)
l1b.parse()
l1b.info.subset_region_name = self.l2def.hemisphere
return l1b
def parse(self):
self._log("Parsing cs2awi grid file: %s" %
filename_from_path(self._filename))
nc = ReadNC(self._filename)
for parameter in nc.parameters:
data = np.ma.array(getattr(nc, parameter))
data.mask = np.isnan(data)
setattr(self, parameter, data)
self._register_parameter(parameter)
self._log("- found parameter: %s dims:%s" %
(parameter, str(np.shape(data))))
def _transfer_metadata(self):
pass
""" Extract essential metadata information from SGDR file """
info = self.l1b.info
sgdr = self.sgdr
info.set_attribute("pysiral_version", PYSIRAL_VERSION)
info.set_attribute("mission", self._mission)
info.set_attribute("mission_data_version", sgdr.nc.software_ver)
info.set_attribute("orbit", sgdr.nc.abs_orbit)
info.set_attribute("cycle", sgdr.nc.cycle)
mission_data_source = filename_from_path(sgdr.nc.filename)
info.set_attribute("mission_data_source", mission_data_source)
info.set_attribute("timeliness", "REP")
def _set_input_file_metadata(self):
""" Fill the product info """
# Short cuts
metadata = self.nc.attrs
info = self.l1.info
# Processing environment metadata
info.set_attribute("pysiral_version", pysiral_version)
# General product metadata
info.set_attribute("mission", "cryosat2")
info.set_attribute("mission_sensor", "siral")
info.set_attribute("mission_data_version", "D")
info.set_attribute("orbit", metadata["abs_orbit_start"])
info.set_attribute("cycle", metadata["cycle_number"])
info.set_attribute("mission_data_source",
filename_from_path(self.filepath))
info.set_attribute(
"timeliness",
cs2_procstage2timeliness(metadata["processing_stage"]))
# Time-Orbit Metadata
lats = [
float(metadata["first_record_lat"]) * 1e-6,
float(metadata["last_record_lat"]) * 1e-6
]
lons = [
float(metadata["first_record_lon"]) * 1e-6,
float(metadata["last_record_lon"]) * 1e-6
]
info.set_attribute("start_time",
parse_datetime_str(
metadata["first_record_time"][4:])) # TAI=....
info.set_attribute("stop_time",
parse_datetime_str(
metadata["last_record_time"][4:])) # TAI=....
info.set_attribute("lat_min", np.amin(lats))
info.set_attribute("lat_max", np.amax(lats))
info.set_attribute("lon_min", np.amin(lons))
info.set_attribute("lon_max", np.amax(lons))
# Product Content Metadata
for mode in ["sar", "sin", "lrm"]:
percent_value = 0.0
if metadata["sir_op_mode"].strip().lower() == mode:
percent_value = 100.
info.set_attribute("{}_mode_percent".format(mode), percent_value)
info.set_attribute("open_ocean_percent",
float(metadata["open_ocean_percent"]) * 0.01)
def parse(self):
self._log("Parsing sicci pickled grid file: %s" %
filename_from_path(self._filename))
G = pickle.load(open(self._filename))
# Extract vector fb values.
self.sea_ice_freeboard = np.ma.array(G.AverageFreeboard(False)[0])
self.sea_ice_freeboard *= 1.0e-3 # in mm!
self.sea_ice_freeboard.mask = np.isnan(self.sea_ice_freeboard)
self.latitude = np.array(G.lat) # in degrees
self.longitude = np.array(G.lon) # in degrees
self._register_parameter("longitude")
self._register_parameter("latitude")
self._register_parameter("sea_ice_freeboard")
def write_to_file(self, output_id, directory):
""" Write a summary file to the defined export directory """
# Create a simple filename
filename = os.path.join(directory, "pysiral-l2proc-summary.txt")
self.log.info("Exporting summary report: %s" % filename)
lfmt = " %-16s : %s\n"
current_time = str(datetime.now()).split(".")[0]
with open(filename, "w") as fhandle:
# Write infos on settings, host, os, ....
fhandle.write("# pysiral Level2Processor Summary\n\n")
fhandle.write(lfmt % ("created", current_time))
# Brief statistics of files, errors, warnings
fhandle.write("\n# Processor Statistics\n\n")
fhandle.write(lfmt % ("l1b files", str(self.n_files)))
fhandle.write(lfmt % ("errors", str(self.n_discarded_files)))
fhandle.write(lfmt % ("warnings", str(self.n_warnings)))
fhandle.write("\n# Processor & Local Machine Settings\n\n")
fhandle.write(lfmt % ("pysiral version", PYSIRAL_VERSION))
fhandle.write(lfmt % ("python version", sys.version))
fhandle.write(lfmt % ("hostname", HOSTNAME))
# More info on this specific run
fhandle.write(lfmt % ("data period", self.data_period_str))
fhandle.write(lfmt % ("Level-2 settings", self.l2_settings_file))
fhandle.write(lfmt % ("l1b repository", self.l1b_repository))
# List discarded files and reason (error code & description)
fhandle.write("\n# Detailed Error Breakdown\n\n")
msg = " No %s output generated for %g l1b files due " + \
"to following errors:\n"
fhandle.write(msg % (output_id, self.n_discarded_files))
for error_code in PYSIRAL_ERROR_CODES.keys():
n_discarded_files = len(self.error_counter[error_code])
if n_discarded_files == 0:
continue
error_description = PYSIRAL_ERROR_CODES[error_code]
msg = "\n %g file(s): [error_code:%s] %s\n" % (
n_discarded_files, error_code, error_description)
fhandle.write(msg)
for discarded_file in self.error_counter[error_code]:
fn = filename_from_path(discarded_file)
fhandle.write(" * %s\n" % fn)
def parse(self):
self._log("Parsing cs2awi grid file: %s" %
filename_from_path(self._filename))
nc = ReadNC(self._filename)
for parameter in nc.parameters:
data = np.ma.array(getattr(nc, parameter))
data.mask = np.isnan(data)
setattr(self, parameter, data)
self._register_parameter(parameter)
self._log("- found parameter: %s dims:%s" %
(parameter, str(np.shape(data))))
# XXX: dirty hack
nan_mask = np.isnan(self.sea_surface_height_anomaly)
self.radar_freeboard.mask = nan_mask
self.sea_ice_freeboard = self.radar_freeboard
def merge_and_export_polar_ocean_subsets(self):
""" loop over remaining files in file list """
log = self.log
n = len(self._l1b_file_list)
if n == 0:
return
l1bdata_stack = []
for i, l1b_file in enumerate(self._l1b_file_list):
# Parse the current file and split into polar ocean segments
log.info("+ [ Processing %g of %g ]" % (i+1, n))
log.info("- Input file: %s" % filename_from_path(l1b_file))
l1b_segments = self.get_ocean_segments_from_input_file(l1b_file)
# Skip if no relevant data was found
if l1b_segments is None:
log.info("- %s polar ocean data: False -> skip file" % (
self._jobdef.hemisphere))
continue
else:
log.info("- %s polar ocean data: True" % (
self._jobdef.hemisphere))
log.info("- %g polar ocean data segments" % len(l1b_segments))
# XXX: Debug
# debug_stack_orbit_plot(l1bdata_stack, l1b_segments)
# Loop over segments and check connectivity
for l1b_segment in l1b_segments:
if not self.l1b_is_connected_to_stack(
l1b_segment, l1bdata_stack):
# => break criterium, save existing stack and start over
log.info("- segment unconnected, exporting current stack")
l1b_merged = self.merge_l1b_stack(l1bdata_stack)
self.export_l1b_to_netcdf(l1b_merged)
# Reset the l1bdata stack
l1bdata_stack = [l1b_segment]
continue
# polar ocean data and connected => add to stack
l1bdata_stack.append(l1b_segment)
def process_l2i_files(self, l2i_files, period):
""" Reads all l2i files and merges the valid data into a l2p
summary file """
# l2p: Container for storing l2i objects
l2p = Level2PContainer(period)
# Add all l2i objects to the l2p container.
# NOTE: Only memory is the limit
for l2i_file in l2i_files:
try:
l2i = L2iNCFileImport(l2i_file)
except Exception, errmsg:
msg = "Error (%s) in l2i file: %s"
msg = msg % (errmsg, filename_from_path(l2i_file))
self.log.error(msg)
continue
l2p.append_l2i(l2i)
def _set_input_file_metadata(self):
""" Extract essential metadata information from SGDR file """
info = self.l1.info
sgdr = self.sgdr
info.set_attribute("pysiral_version", PYSIRAL_VERSION)
try:
info.set_attribute(
"mission", self.cfg.platform_name_dict[str(sgdr.nc.mission)])
except KeyError:
mission_id = self.sgdr.guess_mission_from_filename()
info.set_attribute("mission",
self.cfg.platform_name_dict[str(mission_id)])
info.set_attribute("mission_data_version", sgdr.nc.software_ver)
info.set_attribute("orbit", sgdr.nc.abs_orbit)
info.set_attribute("cycle", sgdr.nc.cycle)
mission_data_source = filename_from_path(sgdr.nc.filename)
info.set_attribute("mission_data_source", mission_data_source)
info.set_attribute("timeliness", self.cfg.timeliness)
def parse_filename(self, fn):
""" Parse info from pysiral output filename """
filename = filename_from_path(fn)
match_found = False
for data_level in self._registered_parsers.keys():
parser = parse.compile(self._registered_parsers[data_level])
match = parser.parse(filename)
if match:
match_found = True
self.data_level = data_level
for parameter in match.named.keys():
value = match[parameter]
if parameter in ["start", "stop"]:
try:
value = dtparser.parse(value)
except:
match_found = False
break
setattr(self, parameter, value)
break
if not match_found:
print "Unrecognized filename: %s" % filename