def get_object_information(self, inobj):
odim_source.CheckSource(inobj)
S = odim_source.ODIM_Source(inobj.source)
try:
return qitotal_options.get_qitotal_site_information(self._qitotal_option_file)[S.nod]
except KeyError:
return qitotal_options.get_qitotal_site_information(self._qitotal_option_file)["default"]
def get_options(inobj):
odim_source.CheckSource(inobj)
S = odim_source.ODIM_Source(inobj.source)
try:
return copy.deepcopy(ARGS[S.nod])
except KeyError:
return copy.deepcopy(ARGS["default"])
def NODfromSourceString(source):
S = odim_source.ODIM_Source(source)
if S.nod: return S.nod
else:
try:
return odim_source.NOD[S.wmo]
except KeyError:
return None
def testODIM_Source(self):
s = odim_source.SOURCE[self.MYNOD]
S = odim_source.ODIM_Source(s.encode("UTF-8"))
self.assertEqual(S.nod, self.MYNOD)
self.assertEqual(S.rad, 'FI51')
self.assertEqual(S.plc, u'Pet\xe4j\xe4vesi')
self.assertEqual(S.source, s.encode("UTF-8"))
self.assertEqual(S.wmo, '02775')
def Validate(rio):
# Zero check: do we have a payload?
if not rio.object:
raise IOError("Failed to read file.")
# First ODIM violation fix: ODIM_BUFR cannot tell the difference between
# a SCAN and a PVOL. ODIM_H5 can.
if rio.objectType == _raveio.Rave_ObjectType_PVOL and rio.object.getNumberOfScans(
) == 1:
DATE, TIME = rio.object.date, rio.object.time
rio.object = rio.object.getScan(0)
rio.object.date, rio.object.time = DATE, TIME
# Second (potential) ODIM violation: bad or incomplete value of /what/source
# Harmonize, because bRopo looks up filter settings based on this attribute.
try:
odim_source.CheckSource(rio.object)
s = odim_source.ODIM_Source(rio.object.source)
if not s.nod: raise AttributeError()
if not s.wmo:
rio.object.source = odim_source.SOURCE[s.nod].encode(UTF8)
except:
rio.object.source = repair_odim_source(rio.object)
s = odim_source.ODIM_Source(rio.object.source)
# Third issue, not an ODIM violation as such: ODIM_BUFR returns datasets as
# float64 arrays, which a massive waste of RAM. Harmonize to uint8.
# Note: this is not a problem for the toolbox as such, but it is a problem
# for Ropo because Ropo assumes uint8 data.
if rio.file_format == _raveio.RaveIO_ODIM_FileFormat_BUFR:
ConvertDatasets(rio)
# Step 4: combination of data and metadata fixes.
# In one known instance, a country supplies TH but not DBZH data. bRopo
# currently requires DBZH, so in these cases rename TH to DBZH.
# Very few countries supply their data violating the ancient GORN "rules"
# for nodata and undetect, but it happens, so identify and fix.
# Note: this is not a problem for the toolbox as such, but it is a problem
# for Ropo because Ropo assumes standard values.
# Unfortunately, all data must be checked.
# While we're at it, perform a sanity check on the values of 'where/rstart'
# and 'where/rscale'. This is an ODIM violation that doesn't need
# correcting for ropo, but should be correctly represented for use by
# other parts of the toolbox if required.
CheckOddsnEnds(rio)
def MakePolarFileName(rio, root=ECROOT, makepath=False, Round=False):
global ECDATA
ECDATA = root
s = odim_source.ODIM_Source(rio.object.source)
if Round:
DATE, TIME = RoundDT(rio.object.date, rio.object.time)
else:
DATE, TIME = rio.object.date, rio.object.time
path = GetPathFromTuple((int(DATE[:4]), int(DATE[4:6]), int(DATE[6:8]),
int(TIME[:2]), int(TIME[2:4]), 0, 0, 0, 0))
if makepath and not os.path.isdir(path):
try:
os.makedirs(path)
except:
sys.stderr.write('Warning: makedirs failed for %s\n' % path)
fstr = os.path.join(path, s.nod)
try:
# Squash any empty spaces
task = re.sub(" ", "", rio.object.getAttribute('how/task').lower())
except AttributeError:
task = None
if task:
fstr = '%s_%s' % (fstr, task)
if rio.objectType == _raveio.Rave_ObjectType_SCAN:
fstr = '%s_%2.1f' % (fstr, rio.object.elangle * rd)
else:
if rio.objectType == _raveio.Rave_ObjectType_SCAN:
fstr = '%s_scan' % fstr
fstr = '%s_%2.1f' % (fstr, rio.object.elangle * rd)
elif rio.objectType == _raveio.Rave_ObjectType_PVOL:
fstr = '%s_pvol' % fstr
hexq = makeHexQuant(rio)
# fstr = '%s_%sT%sZ.h5' % (fstr, DATE, TIME[:-2]) # Time the script is run.
# fstr = '%s_%sT%s%sZ_%s.h5' % (fstr, DATE, TIME[:2], path.split("/")[-2], hexq)
fstr = '%s_%sT%sZ_%s.h5' % (fstr, rio.object.date, rio.object.time[:4],
hexq)
return fstr
def _add_radar_index_value_to_argument_list(self, args):
ctr = 1
for k in self.file_objects.keys():
v = self.file_objects[k]
if not _polarscan.isPolarScan(
v) and not _polarvolume.isPolarVolume(v):
continue
sourceid = v.source
try:
osource = odim_source.ODIM_Source(v.source)
if osource.wmo:
sourceid = "WMO:%s" % osource.wmo
elif osource.rad:
sourceid = "RAD:%s" % osource.rad
elif osource.nod:
sourceid = "NOD:%s" % osource.nod
except:
pass
for arg in args:
arg[0].radar_index_mapping[sourceid] = ctr
ctr = ctr + 1
def _generate(self, dd, dt, area=None):
self._debug_generate_info(area)
if self.verbose:
self.logger.info("Fetching objects and applying quality plugins")
self.logger.debug(
"Generating composite with date and time %sT%s for area %s", dd,
dt, area)
objects, nodes, how_tasks, all_files_malfunc = self.fetch_objects()
if all_files_malfunc:
self.logger.info(
"Content of all provided files were marked as 'malfunc'. Since option 'ignore_malfunc' is set, no composite is generated!"
)
return None
objects, algorithm, qfields = self.quality_control_objects(objects)
self.logger.debug("Quality controls for composite generation: %s",
(",".join(qfields)))
if len(objects) == 0:
self.logger.info(
"No objects provided to the composite generator. No composite will be generated!"
)
return None
objects = list(objects.values())
if self.dump:
self._dump_objects(objects)
generator = _pycomposite.new()
if area is not None:
if _area.isArea(area):
pyarea = area
else:
pyarea = my_area_registry.getarea(area)
else:
if self.verbose:
self.logger.info("Determining best fit for area")
A = rave_area.MakeAreaFromPolarObjects(objects, self.pcsid,
self.xscale, self.yscale)
pyarea = _area.new()
pyarea.id = "auto-generated best-fit"
pyarea.xsize = A.xsize
pyarea.ysize = A.ysize
pyarea.xscale = A.xscale
pyarea.yscale = A.yscale
pyarea.extent = A.extent
pcs = rave_projection.pcs(A.pcs)
pcsname = pcs.name
if not is_py27:
pcsname = pcsname.decode()
pyarea.projection = _projection.new(pcs.id, pcsname,
' '.join(pcs.definition))
if len(objects) == 1:
try:
tmpid = odim_source.NODfromSource(objects[0])
pyarea.id = "auto_%s_%s" % (A.pcs, tmpid)
except:
pass
generator.addParameter(self.quantity, self.gain, self.offset,
self.minvalue)
generator.product = self.product
if algorithm is not None:
generator.algorithm = algorithm
for o in objects:
generator.add(o)
# We want to ensure that we get a proper indexing of included radar
sourceid = o.source
try:
osource = odim_source.ODIM_Source(o.source)
if osource.wmo:
sourceid = "WMO:%s" % osource.wmo
elif osource.rad:
sourceid = "RAD:%s" % osource.rad
elif osource.nod:
sourceid = "NOD:%s" % osource.nod
except:
pass
if not sourceid in self.radar_index_mapping.keys():
self.radar_index_mapping[sourceid] = self.get_next_radar_index(
)
generator.selection_method = self.selection_method
generator.interpolation_method = self.interpolation_method
generator.date = o.date if dd is None else dd
generator.time = o.time if dt is None else dt
generator.height = self.height
generator.elangle = self.elangle
generator.range = self.range
if self.qitotal_field is not None:
generator.quality_indicator_field_name = self.qitotal_field
if self.prodpar is not None:
self._update_generator_with_prodpar(generator)
if self.verbose:
self.logger.info("Generating cartesian composite")
generator.applyRadarIndexMapping(self.radar_index_mapping)
result = generator.generate(pyarea, qfields)
if self.applyctfilter:
if self.verbose:
self.logger.debug("Applying ct filter")
rave_ctfilter.ctFilter(result, self.quantity)
if self.applygra:
if not "se.smhi.composite.distance.radar" in qfields:
self.logger.info(
"Trying to apply GRA analysis without specifying a quality plugin specifying the se.smhi.composite.distance.radar q-field, disabling..."
)
else:
if self.verbose:
self.logger.info(
"Applying GRA analysis (ZR A = %f, ZR b = %f)" %
(self.zr_A, self.zr_b))
grafield = self._apply_gra(result, dd, dt)
if grafield:
result.addParameter(grafield)
else:
self.logger.warn("Failed to generate gra field....")
# Hack to create a BRDR field if the qfields contains se.smhi.composite.index.radar
if "se.smhi.composite.index.radar" in qfields:
bitmapgen = _bitmapgenerator.new()
brdr_field = bitmapgen.create_intersect(
result.getParameter(self.quantity),
"se.smhi.composite.index.radar")
brdr_param = result.createParameter("BRDR",
_rave.RaveDataType_UCHAR)
brdr_param.setData(brdr_field.getData())
if self.applygapfilling:
if self.verbose:
self.logger.debug("Applying gap filling")
t = _transform.new()
gap_filled = t.fillGap(result)
result.getParameter(self.quantity).setData(
gap_filled.getParameter(self.quantity).getData())
# Fix so that we get a valid place for /what/source and /how/nodes
plc = result.source
result.source = "%s,CMT:%s" % (CENTER_ID, plc)
result.addAttribute('how/nodes', nodes)
if self.use_site_source and len(objects) == 1:
try:
result.source = objects[0].source
if result.source.find("NOD:") == -1:
tmpid = odim_source.NODfromSource(objects[0])
result.source = "%s,NOD:%s,CMT:%s" % (
self.remove_CMT_from_source(result.source), tmpid, plc)
else:
result.source = "%s,CMT:%s" % (self.remove_CMT_from_source(
result.source), plc)
except:
self.logger.exception("Failed to get source from object")
if how_tasks != "":
result.addAttribute('how/task', how_tasks)
if self.verbose:
self.logger.debug("Returning resulting composite image")
return result
def repair_odim_source(obj):
import re
source = obj.source
# France - Falaise
if obj.source == 'WMO:07027':
return odim_source.SOURCE["frcae"].encode(UTF8)
# France - Nancy
elif obj.source == 'WMO:07180':
return odim_source.SOURCE["frnan"].encode(UTF8)
# France - Nimes
elif obj.source == 'WMO:07645':
return odim_source.SOURCE["frnim"].encode(UTF8)
# France - Arcis
elif obj.source == 'WMO:07168':
return odim_source.SOURCE["frtro"].encode(UTF8)
# France - Sembadel
elif obj.source == 'WMO:07471':
return odim_source.SOURCE["frlep"].encode(UTF8)
# Cerceda - Spain
elif obj.source[:9] == 'WMO:08007':
return odim_source.SOURCE["escor"].encode(UTF8)
# Almeria - Spain
elif obj.source[:9] == 'WMO:08489':
return odim_source.SOURCE["esalm"].encode(UTF8)
# Spain - Sierra de Fuentes, a k a Badajoz
elif obj.source[:9] == "WMO:08262":
return odim_source.SOURCE["esbad"].encode(UTF8)
# Spain - Autilla Pino, a k a Valladolid
elif obj.source[:9] == "WMO:08262":
return odim_source.SOURCE["eslid"].encode(UTF8)
# Spain - Aguion, a k a Santander
elif obj.source[:9] == "WMO:08019":
return odim_source.SOURCE["essan"].encode(UTF8)
# Jersey
elif obj.source == 'WMO:00897':
return odim_source.SOURCE["ukjer"].encode(UTF8)
# Keflavik
elif obj.source == 'WMO:0,PLC:Keflavik':
obj.height = 45.0
return odim_source.SOURCE["iskef"].encode(UTF8)
# Dublin
elif obj.source == 'WMO:03696':
return odim_source.SOURCE["iedub"].encode(UTF8)
# Essen
elif obj.source == 'WMO:10412':
return odim_source.SOURCE["deess"].encode(UTF8)
# KNMI:
elif len(source.split(";")) == 2 or source[:6] == "RAD:NL":
#source = re.sub(";", ",", source) # no longer needed
source = re.sub("PLC", "NOD", source)
s = odim_source.ODIM_Source(source)
return odim_source.SOURCE[s.nod].encode(UTF8)
# met.no
elif re.search("1438", obj.source):
return odim_source.SOURCE["nohgb"].encode(UTF8)
elif re.search("1104", obj.source):
return odim_source.SOURCE["norst"].encode(UTF8)
elif re.search("1405", obj.source):
return odim_source.SOURCE["nobml"].encode(UTF8)
elif obj.source == 'WMO:01079,NOD:nober,PLC:Berlev\xe5g':
return odim_source.SOURCE["nober"].encode(UTF8)
# Serbia
elif obj.source == "WMO:0,PLC:Samos":
return odim_source.SOURCE["rssam"].encode(UTF8)
# Iceland
elif obj.source == "WMO:0,PLC:Teigsbjarg":
return odim_source.SOURCE["istgb"].encode(UTF8)
