def fetch_objects(self):
nodes = ""
objects = {}
tasks = []
for fname in self.filenames:
obj = None
try:
if self.ravebdb != None:
obj = self.ravebdb.get_rave_object(fname)
else:
obj = _raveio.open(fname).object
except IOError:
self.logger.exception("Failed to open %s" % fname)
is_scan = _polarscan.isPolarScan(obj)
if is_scan:
is_pvol = False
else:
is_pvol = _polarvolume.isPolarVolume(obj)
if not is_scan and not is_pvol:
self.logger.info(
"Input file %s is neither polar scan or volume, ignoring."
% fname)
continue
if self.ignore_malfunc:
obj = rave_util.remove_malfunc(obj)
if obj is None:
continue
node = odim_source.NODfromSource(obj)
if len(nodes):
nodes += ",'%s'" % node
else:
nodes += "'%s'" % node
objects[fname] = obj
if is_scan:
self.logger.debug(
"Scan used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
self.add_how_task_from_scan(obj, tasks)
elif is_pvol:
self.logger.debug(
"PVOL used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
self.add_how_task_from_scan(scan, tasks)
how_tasks = ",".join(tasks)
return objects, nodes, how_tasks
def generate_new_volume_with_qc(original_file, args):
logger.debug("Generating new volume with quality controls applied.")
if ravebdb != None:
volume = ravebdb.get_rave_object(original_file)
else:
volume = _raveio.open(original_file).object
if "remove-malfunc" in args.keys():
try:
if args["remove-malfunc"].lower() in ["true", "yes", "y", "1"]:
logger.debug(
"Checking volume for malfunc tags. Will remove scans, or complete volume, if marked malfunc."
)
volume = rave_util.remove_malfunc(volume)
if volume == None:
logger.info(
"Malfunc volume! Since option 'remove_malfunc' is set, no new volume with QC applied will be generated!"
)
return None
except:
pass
if "anomaly-qc" in args.keys():
detectors = args["anomaly-qc"].split(",")
else:
detectors = []
quality_control_mode = QUALITY_CONTROL_MODE_ANALYZE_AND_APPLY
if "qc-mode" in args.keys():
quality_control_mode = args["qc-mode"]
volume = perform_quality_control(volume, detectors, quality_control_mode)
logger.debug("Quality controls applied on new volume: %s",
(",".join(detectors)))
new_time = args.get('time')
if new_time:
volume.time = new_time
new_date = args.get('date')
if new_date:
volume.date = new_date
return volume
def test_remove_malfunc__all_malfunc(self):
pv = _polarvolume.new()
pv.addAttribute("how/malfunc", 'False')
ps1 = _polarscan.new()
ps1.addAttribute("how/malfunc", 'True')
ps1.elangle = 1.0
ps2 = _polarscan.new()
ps2.addAttribute("how/malfunc", 'True')
ps2.elangle = 2.0
ps3 = _polarscan.new()
ps3.addAttribute("how/malfunc", 'True')
ps3.elangle = 3.0
pv.addScan(ps1)
pv.addScan(ps2)
pv.addScan(ps3)
result = rave_util.remove_malfunc(pv)
self.assertTrue(result == None)
def test_remove_malfunc(self):
pv = _polarvolume.new()
pv.addAttribute("how/malfunc", 'False')
ps1 = _polarscan.new()
ps1.addAttribute("how/malfunc", 'False')
ps1.elangle = 1.0
ps2 = _polarscan.new()
ps2.addAttribute("how/malfunc", 'True')
ps2.elangle = 2.0
ps3 = _polarscan.new()
ps3.addAttribute("how/malfunc", 'False')
ps3.elangle = 3.0
pv.addScan(ps1)
pv.addScan(ps2)
pv.addScan(ps3)
result = rave_util.remove_malfunc(pv)
self.assertTrue(result == pv)
self.assertEqual(2, result.getNumberOfScans())
self.assertTrue(ps1 == pv.getScan(0))
self.assertTrue(ps3 == pv.getScan(1))
def test_remove_malfunc_4(self):
c = _cartesian.new()
c.addAttribute("how/malfunc", 'True')
result = rave_util.remove_malfunc(c)
self.assertTrue(result == c)
def test_remove_malfunc_3(self):
ps = _polarscan.new()
ps.addAttribute("how/malfunc", 'True')
result = rave_util.remove_malfunc(ps)
self.assertTrue(result is None)
def fetch_objects(self):
nodes = ""
objects = {}
tasks = []
malfunc_files = 0
preload_quantity = None
if self.use_lazy_loading and self.use_lazy_loading_preloads:
preload_quantity = self.quantity
for fname in self.filenames:
obj = None
try:
if self.ravebdb != None:
obj = self.ravebdb.get_rave_object(fname,
self.use_lazy_loading,
preload_quantity)
else:
obj = _raveio.open(fname, self.use_lazy_loading,
preload_quantity).object #self.quantity
except IOError:
self.logger.exception("Failed to open %s", fname)
is_scan = _polarscan.isPolarScan(obj)
if is_scan:
is_pvol = False
else:
is_pvol = _polarvolume.isPolarVolume(obj)
if not is_scan and not is_pvol:
self.logger.warn(
"Input file %s is neither polar scan or volume, ignoring.",
fname)
continue
# Force azimuthal nav information usage if requested
obj.use_azimuthal_nav_information = self.use_azimuthal_nav_information
if self.ignore_malfunc:
obj = rave_util.remove_malfunc(obj)
if obj is None:
self.logger.info(
"Input file %s detected as 'malfunc', ignoring.",
fname)
malfunc_files += 1
continue
node = odim_source.NODfromSource(obj)
if len(nodes):
nodes += ",'%s'" % node
else:
nodes += "'%s'" % node
objects[fname] = obj
if is_scan:
self.logger.debug(
"Scan used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
self.add_how_task_from_scan(obj, tasks)
elif is_pvol:
self.logger.debug(
"PVOL used in composite generation - UUID: %s, Node: %s, Nominal date and time: %sT%s",
fname, node, obj.date, obj.time)
for i in range(obj.getNumberOfScans()):
scan = obj.getScan(i)
self.add_how_task_from_scan(scan, tasks)
how_tasks = ",".join(tasks)
all_files_malfunc = (len(self.filenames) > 0
and malfunc_files == len(self.filenames))
return objects, nodes, how_tasks, all_files_malfunc
