def test_ctoscan(self):
obj = _transform.new()
cartesian = _raveio.open(self.FIXTURE_CARTESIAN_PPI).object
volume = _raveio.open(self.FIXTURE_VOLUME).object
radardef = _radardef.new()
radardef.id = volume.source
radardef.description = "ctop test"
radardef.longitude = volume.longitude
radardef.latitude = volume.latitude
radardef.height = volume.height
elangles = []
nangles = volume.getNumberOfScans()
for i in range(nangles):
scan = volume.getScan(i)
elangles.append(scan.elangle)
radardef.elangles = elangles
scan = volume.getScan(0)
radardef.nrays = scan.nrays
radardef.nbins = scan.nbins
radardef.scale = scan.rscale
radardef.beamwidth = scan.beamwidth
elangle = scan.elangle
result = obj.ctoscan(cartesian, radardef, elangle, "DBZH")
rio = _raveio.new()
rio.object = result
rio.save("ctop_polarscan.h5")
def testFillGap_onCartesianParameters(self):
data = numpy.zeros((6, 6), numpy.uint8)
data[1][2] = 1
data[2][1] = 1
data[3][2] = 1
data[2][3] = 1
obj = _cartesian.new()
param = _cartesianparam.new()
param.setData(data)
param.nodata = 255.0
param.quantity = "DBZH"
obj.addParameter(param)
param = _cartesianparam.new()
data[1][2] = 2
data[2][1] = 2
data[3][2] = 2
data[2][3] = 2
param.setData(data)
param.nodata = 255.0
param.quantity = "TH"
obj.addParameter(param)
t = _transform.new()
result = t.fillGap(obj)
data = result.getParameter("DBZH").getData()
self.assertEqual(1, data[2][2])
data = result.getParameter("TH").getData()
self.assertEqual(2, data[2][2])
def test_fillGap(self):
obj = _transform.new()
io = _raveio.new()
cartesian = _raveio.open(self.FIXTURE_CARTESIAN_PCAPPI).object
io.object = obj.fillGap(cartesian)
io.filename = self.TRANSFORM_FILLGAP_FILENAME
io.save()
def testValidMethods(self):
obj = _transform.new()
meths = [
_rave.NEAREST, _rave.BILINEAR, _rave.CUBIC, _rave.CRESSMAN,
_rave.UNIFORM, _rave.INVERSE
]
for method in meths:
obj.method = method
self.assertEqual(method, obj.method)
def testInvalidMethods(self):
obj = _transform.new()
meths = [99, 33, 22, 11]
for method in meths:
try:
obj.method = method
self.fail("Expected ValueError")
except ValueError:
pass
self.assertEqual(_rave.NEAREST, obj.method)
def testFillGap_onParameter(self):
data = numpy.zeros((6, 6), numpy.uint8)
data[1][2] = 1
data[2][1] = 1
data[3][2] = 1
data[2][3] = 1
param = _cartesianparam.new()
param.setData(data)
param.nodata = 255.0
t = _transform.new()
result = t.fillGap(param)
data = result.getData()
self.assertEqual(1, data[2][2])
def generate(self, dd, dt, area=None):
pyarea = my_area_registry.getarea(area)
if self.preprocess_qc and self.mp_process_qc and self.number_of_quality_control_processes > 1:
self.file_objects, self.nodes, self.how_tasks, all_files_malfunc = self._fetch_file_objects_mp(
)
else:
self.file_objects, self.nodes, self.how_tasks, all_files_malfunc = self._fetch_file_objects(
)
if all_files_malfunc:
self.logger.info(
"Content of all provided files were marked as 'malfunc'. Since option 'ignore_malfunc' is set, no composite is generated!"
)
return None
args = self._create_arguments(dd, dt, pyarea)
results = []
ntiles = len(args)
ncpucores = multiprocessing.cpu_count()
nrprocesses = ntiles
if not RAVE_TILE_COMPOSITING_PROCESSES is None:
if nrprocesses > RAVE_TILE_COMPOSITING_PROCESSES:
nrprocesses = RAVE_TILE_COMPOSITING_PROCESSES
if nrprocesses > ncpucores:
nrprocesses = ncpucores
if nrprocesses == ncpucores and ncpucores > 1:
nrprocesses = nrprocesses - 1 # We always want to leave at least one core for something else
pool = multiprocessing.Pool(nrprocesses)
r = pool.map_async(comp_generate, args, callback=results.append)
r.wait()
pool.terminate()
pool.join()
self.logger.info("Finished processing tiles, combining tiles")
objects = []
try:
for v in results[0]:
tile_file = v[1]
if tile_file == None:
self.logger.warn(
"No partial composite for tile area %s was created. This tile will therefore not be included in complete composite.",
v[0])
else:
o = _raveio.open(tile_file).object
if _cartesianvolume.isCartesianVolume(o):
o = o.getImage(0)
o.objectType = _rave.Rave_ObjectType_COMP
objects.append(o)
t = _transform.new()
self.logger.debug(
"Combining %d tiles into one composite for area %s.",
len(objects), area)
result = t.combine_tiles(pyarea, objects)
# Fix so that we get a valid place for /what/source and /how/nodes
result.source = "%s,CMT:%s" % (CENTER_ID, area)
result.addAttribute('how/nodes', self.nodes)
if self.how_tasks != "":
result.addAttribute('how/task', self.how_tasks)
self.logger.info("Tiles combined")
return result
finally:
if self._do_remove_temporary_files:
for fname in self.compositing.filenames:
try:
os.unlink(fname)
except:
logger.info("Failed to remove temporary file: %s" %
fname)
if results != None:
for v in results[0]:
if v != None and v[1] != None and os.path.exists(v[1]):
try:
os.unlink(v[1])
except Exception:
logger.exception("Failed to unlink %s" % v[1])
return None
def testMethod(self):
obj = _transform.new()
self.assertEqual(_rave.NEAREST, obj.method)
obj.method = _rave.CUBIC
self.assertEqual(_rave.CUBIC, obj.method)
def test_attribute_visibility(self):
attrs = ['method']
obj = _transform.new()
alist = dir(obj)
for a in attrs:
self.assertEqual(True, a in alist)
def test_new(self):
obj = _transform.new()
self.assertNotEqual(-1, str(type(obj)).find("TransformCore"))
def test_combine_tiles_with_two_parameters(self):
pyarea = _area.new()
pyarea.extent = (971337.728807, 7196461.17902, 3015337.72881,
11028461.179)
pyarea.xscale = 511000.0
pyarea.yscale = 958000.0
pyarea.xsize = 4
pyarea.ysize = 4
pyarea.projection = _projection.new(
"x", "y",
"+proj=merc +lat_ts=0 +lon_0=0 +k=1.0 +R=6378137.0 +nadgrids=@null +no_defs"
)
ul = self.create_cartesian_with_parameter(
2, 2, pyarea.xscale, pyarea.yscale,
(971337.728807, 9112461.1790100001, 1993337.7288084999,
11028461.179), pyarea.projection.definition, numpy.uint8, [2, 12],
["DBZH", "TH"])
ul.getParameter("DBZH").addQualityField(
self.create_quality_field(2, 2, numpy.uint8, 22,
"se.some.how.task.1"))
ul.getParameter("TH").addQualityField(
self.create_quality_field(2, 2, numpy.uint8, 32,
"se.some.how.task.1"))
ur = self.create_cartesian_with_parameter(
2, 2, pyarea.xscale, pyarea.yscale,
(1993337.7288084999, 9112461.1790100001, 3015337.72881,
11028461.179), pyarea.projection.definition, numpy.uint8, [3, 13],
["DBZH", "TH"])
ur.getParameter("DBZH").addQualityField(
self.create_quality_field(2, 2, numpy.uint8, 23,
"se.some.how.task.1"))
ur.getParameter("TH").addQualityField(
self.create_quality_field(2, 2, numpy.uint8, 33,
"se.some.how.task.1"))
ll = self.create_cartesian_with_parameter(
2, 2, pyarea.xscale, pyarea.yscale,
(971337.728807, 7196461.17902, 1993337.7288084999,
9112461.1790100001), pyarea.projection.definition, numpy.uint8,
[4, 14], ["DBZH", "TH"])
ll.getParameter("DBZH").addQualityField(
self.create_quality_field(2, 2, numpy.uint8, 24,
"se.some.how.task.1"))
ll.getParameter("TH").addQualityField(
self.create_quality_field(2, 2, numpy.uint8, 34,
"se.some.how.task.1"))
lr = self.create_cartesian_with_parameter(
2, 2, pyarea.xscale, pyarea.yscale,
(1993337.7288084999, 7196461.17902, 3015337.72881,
9112461.1790100001), pyarea.projection.definition, numpy.uint8,
[5, 15], ["DBZH", "TH"])
lr.getParameter("DBZH").addQualityField(
self.create_quality_field(2, 2, numpy.uint8, 25,
"se.some.how.task.1"))
lr.getParameter("TH").addQualityField(
self.create_quality_field(2, 2, numpy.uint8, 35,
"se.some.how.task.1"))
t = _transform.new()
result = t.combine_tiles(pyarea, [ul, ur, ll, lr])
self.assertEqual(4, result.xsize)
self.assertEqual(4, result.ysize)
self.assertEqual(511000.0, result.xscale, 4)
self.assertEqual(958000.0, result.yscale, 4)
self.assertEqual(971337.728807, result.areaextent[0], 4)
self.assertEqual(7196461.17902, result.areaextent[1], 4)
self.assertEqual(3015337.72881, result.areaextent[2], 4)
self.assertEqual(11028461.179, result.areaextent[3], 4)
self.assertEqual(
"+proj=merc +lat_ts=0 +lon_0=0 +k=1.0 +R=6378137.0 +nadgrids=@null +no_defs",
result.projection.definition)
self.assertEqual(ul.date, result.date)
self.assertEqual(ul.time, result.time)
self.assertEqual(ul.startdate, result.startdate)
self.assertEqual(ul.starttime, result.starttime)
self.assertEqual(ul.enddate, result.enddate)
self.assertEqual(ul.endtime, result.endtime)
self.assertEqual(ul.product, result.product)
self.assertEqual(ul.objectType, result.objectType)
param = result.getParameter("DBZH")
self.assertEqual(
[[2, 2, 3, 3], [2, 2, 3, 3], [4, 4, 5, 5], [4, 4, 5, 5]],
param.getData().tolist())
self.assertEqual([[22, 22, 23, 23], [22, 22, 23, 23], [24, 24, 25, 25],
[24, 24, 25, 25]],
param.getQualityFieldByHowTask(
"se.some.how.task.1").getData().tolist())
param = result.getParameter("TH")
self.assertEqual([[12, 12, 13, 13], [12, 12, 13, 13], [14, 14, 15, 15],
[14, 14, 15, 15]],
param.getData().tolist())
self.assertEqual([[32, 32, 33, 33], [32, 32, 33, 33], [34, 34, 35, 35],
[34, 34, 35, 35]],
param.getQualityFieldByHowTask(
"se.some.how.task.1").getData().tolist())
def _generate(self, dd, dt, area=None):
self._debug_generate_info(area)
if self.verbose:
self.logger.info("Fetching objects and applying quality plugins")
self.logger.debug(
"Generating composite with date and time %sT%s for area %s", dd,
dt, area)
objects, nodes, how_tasks, all_files_malfunc = self.fetch_objects()
if all_files_malfunc:
self.logger.info(
"Content of all provided files were marked as 'malfunc'. Since option 'ignore_malfunc' is set, no composite is generated!"
)
return None
objects, algorithm, qfields = self.quality_control_objects(objects)
self.logger.debug("Quality controls for composite generation: %s",
(",".join(qfields)))
if len(objects) == 0:
self.logger.info(
"No objects provided to the composite generator. No composite will be generated!"
)
return None
objects = list(objects.values())
if self.dump:
self._dump_objects(objects)
generator = _pycomposite.new()
if area is not None:
if _area.isArea(area):
pyarea = area
else:
pyarea = my_area_registry.getarea(area)
else:
if self.verbose:
self.logger.info("Determining best fit for area")
A = rave_area.MakeAreaFromPolarObjects(objects, self.pcsid,
self.xscale, self.yscale)
pyarea = _area.new()
pyarea.id = "auto-generated best-fit"
pyarea.xsize = A.xsize
pyarea.ysize = A.ysize
pyarea.xscale = A.xscale
pyarea.yscale = A.yscale
pyarea.extent = A.extent
pcs = rave_projection.pcs(A.pcs)
pcsname = pcs.name
if not is_py27:
pcsname = pcsname.decode()
pyarea.projection = _projection.new(pcs.id, pcsname,
' '.join(pcs.definition))
if len(objects) == 1:
try:
tmpid = odim_source.NODfromSource(objects[0])
pyarea.id = "auto_%s_%s" % (A.pcs, tmpid)
except:
pass
generator.addParameter(self.quantity, self.gain, self.offset,
self.minvalue)
generator.product = self.product
if algorithm is not None:
generator.algorithm = algorithm
for o in objects:
generator.add(o)
# We want to ensure that we get a proper indexing of included radar
sourceid = o.source
try:
osource = odim_source.ODIM_Source(o.source)
if osource.wmo:
sourceid = "WMO:%s" % osource.wmo
elif osource.rad:
sourceid = "RAD:%s" % osource.rad
elif osource.nod:
sourceid = "NOD:%s" % osource.nod
except:
pass
if not sourceid in self.radar_index_mapping.keys():
self.radar_index_mapping[sourceid] = self.get_next_radar_index(
)
generator.selection_method = self.selection_method
generator.interpolation_method = self.interpolation_method
generator.date = o.date if dd is None else dd
generator.time = o.time if dt is None else dt
generator.height = self.height
generator.elangle = self.elangle
generator.range = self.range
if self.qitotal_field is not None:
generator.quality_indicator_field_name = self.qitotal_field
if self.prodpar is not None:
self._update_generator_with_prodpar(generator)
if self.verbose:
self.logger.info("Generating cartesian composite")
generator.applyRadarIndexMapping(self.radar_index_mapping)
result = generator.generate(pyarea, qfields)
if self.applyctfilter:
if self.verbose:
self.logger.debug("Applying ct filter")
rave_ctfilter.ctFilter(result, self.quantity)
if self.applygra:
if not "se.smhi.composite.distance.radar" in qfields:
self.logger.info(
"Trying to apply GRA analysis without specifying a quality plugin specifying the se.smhi.composite.distance.radar q-field, disabling..."
)
else:
if self.verbose:
self.logger.info(
"Applying GRA analysis (ZR A = %f, ZR b = %f)" %
(self.zr_A, self.zr_b))
grafield = self._apply_gra(result, dd, dt)
if grafield:
result.addParameter(grafield)
else:
self.logger.warn("Failed to generate gra field....")
# Hack to create a BRDR field if the qfields contains se.smhi.composite.index.radar
if "se.smhi.composite.index.radar" in qfields:
bitmapgen = _bitmapgenerator.new()
brdr_field = bitmapgen.create_intersect(
result.getParameter(self.quantity),
"se.smhi.composite.index.radar")
brdr_param = result.createParameter("BRDR",
_rave.RaveDataType_UCHAR)
brdr_param.setData(brdr_field.getData())
if self.applygapfilling:
if self.verbose:
self.logger.debug("Applying gap filling")
t = _transform.new()
gap_filled = t.fillGap(result)
result.getParameter(self.quantity).setData(
gap_filled.getParameter(self.quantity).getData())
# Fix so that we get a valid place for /what/source and /how/nodes
plc = result.source
result.source = "%s,CMT:%s" % (CENTER_ID, plc)
result.addAttribute('how/nodes', nodes)
if self.use_site_source and len(objects) == 1:
try:
result.source = objects[0].source
if result.source.find("NOD:") == -1:
tmpid = odim_source.NODfromSource(objects[0])
result.source = "%s,NOD:%s,CMT:%s" % (
self.remove_CMT_from_source(result.source), tmpid, plc)
else:
result.source = "%s,CMT:%s" % (self.remove_CMT_from_source(
result.source), plc)
except:
self.logger.exception("Failed to get source from object")
if how_tasks != "":
result.addAttribute('how/task', how_tasks)
if self.verbose:
self.logger.debug("Returning resulting composite image")
return result
def _generate(self, dd, dt, area=None):
self._debug_generate_info(area)
if self.verbose:
self.logger.info("Fetching objects and applying quality plugins")
self.logger.debug(
"Generating composite with date and time %sT%s for area %s", dd,
dt, area)
objects, nodes, how_tasks = self.fetch_objects()
objects, algorithm, qfields = self.quality_control_objects(objects)
objects = objects.values()
if self.dump:
self._dump_objects(objects)
generator = _pycomposite.new()
if area is not None:
if _area.isArea(area):
pyarea = area
else:
pyarea = my_area_registry.getarea(area)
else:
if self.verbose:
self.logger.info("Determining best fit for area")
A = rave_area.MakeAreaFromPolarObjects(objects, self.pcsid,
self.xscale, self.yscale)
pyarea = _area.new()
pyarea.id = "auto-generated best-fit"
pyarea.xsize = A.xsize
pyarea.ysize = A.ysize
pyarea.xscale = A.xscale
pyarea.yscale = A.yscale
pyarea.extent = A.extent
pcs = rave_projection.pcs(A.pcs)
pyarea.projection = _projection.new(
pcs.id, pcs.name, string.join(pcs.definition, ' '))
if len(objects) == 1:
try:
tmpid = odim_source.NODfromSource(objects[0])
pyarea.id = "auto_%s_%s" % (A.pcs, tmpid)
except:
pass
if type(self.quantity) is types.StringType:
generator.addParameter(self.quantity, self.gain, self.offset)
else:
for quantity in self.quantity:
generator.addParameter(quantity, self.gain, self.offset)
generator.product = self.product
if algorithm is not None:
generator.algorithm = algorithm
if len(objects) == 0:
self.logger.info("No objects provided to the composite generator.")
if dd is None or dt is None:
self.logger.error(
"Can not create a composite without specifying a valid date / time when no objects are provided."
)
raise Exception, "Can not create a composite without specifying a valid date / time when no objects are provided."
for o in objects:
generator.add(o)
generator.selection_method = self.selection_method
generator.date = o.date if dd is None else dd
generator.time = o.time if dt is None else dt
generator.height = self.height
generator.elangle = self.elangle
generator.range = self.range
if self.qitotal_field is not None:
generator.quality_indicator_field_name = self.qitotal_field
if self.prodpar is not None:
self._update_generator_with_prodpar(generator)
if self.verbose:
self.logger.info("Generating cartesian composite")
result = generator.nearest(pyarea, qfields)
if self.applyctfilter:
if self.verbose:
self.logger.debug("Applying ct filter")
ret = rave_ctfilter.ctFilter(result, self.quantity)
if self.applygra:
if not "se.smhi.composite.distance.radar" in qfields:
self.logger.info(
"Trying to apply GRA analysis without specifying a quality plugin specifying the se.smhi.composite.distance.radar q-field, disabling..."
)
else:
if self.verbose:
self.logger.info(
"Applying GRA analysis (ZR A = %f, ZR b = %f)" %
(self.zr_A, self.zr_b))
grafield = self._apply_gra(result, dd, dt)
if grafield:
result.addParameter(grafield)
else:
self.logger.warn("Failed to generate gra field....")
if self.applygapfilling:
if self.verbose:
self.logger.debug("Applying gap filling")
t = _transform.new()
gap_filled = t.fillGap(result)
result.getParameter(self.quantity).setData(
gap_filled.getParameter(self.quantity).getData())
# Fix so that we get a valid place for /what/source and /how/nodes
plc = result.source
result.source = "%s,CMT:%s" % (CENTER_ID, plc)
result.addAttribute('how/nodes', nodes)
if self.use_site_source and len(objects) == 1:
try:
result.source = objects[0].source
if result.source.find("NOD:") == -1:
tmpid = odim_source.NODfromSource(objects[0])
result.source = "%s,NOD:%s,CMT:%s" % (
self.remove_CMT_from_source(result.source), tmpid, plc)
else:
result.source = "%s,CMT:%s" % (self.remove_CMT_from_source(
result.source), plc)
except:
self.logger.exception("Failed to get source from object")
if how_tasks != "":
result.addAttribute('how/task', how_tasks)
if self.verbose:
self.logger.debug("Returning resulting composite image")
return result