def _reject_all(self): while self._msg_buffer: msg = self._msg_buffer.pop() try: msg.reject() except: log.critical('Failed to reject message')
def _ack_all(self): while self._msg_buffer: msg = self._msg_buffer.pop() try: msg.ack() except: log.critical('Failed to ack message')
def _reject_all(self): while self._msg_buffer: msg = self._msg_buffer.pop() try: msg.reject() except: log.critical('Failed to reject message')
def _ack_all(self): while self._msg_buffer: msg = self._msg_buffer.pop() try: msg.ack() except: log.critical('Failed to ack message')
def get_coverage_doctor(self, dataset_id, data_product_id=None): # Get the associated objects required for rebuilding dset_obj = self.get_dataset_object(dataset_id) if data_product_id is None: # Go find the first data product associated with dataset_id data_product_id, _ = self.resource_registry.find_subjects(object=dataset_id, predicate=PRED.hasDataset, id_only=True) data_product_id = data_product_id[0] if len(data_product_id) > 0 else None if data_product_id is None: raise ValueError('Cannot find any Data Products associated with dataset_id \'{0}\''.format(dataset_id)) dprod_obj = self.get_data_product_object(data_product_id) # Get the path to the editible coverage - also ensures ingestion is paused cpth = None try: with self.get_editable_coverage(dataset_id) as cov: cpth = cov.persistence_dir except IOError, ex: fs = 'Unable to open reference coverage: \'' accessibility_errors = [ 'unable to create file (File accessability: Unable to open file)', 'unable to open file (File accessibilty: Unable to open file)', 'unable to open file (File accessability: Unable to open file)'] if fs in ex.message: # The view coverage couldn't load it's underlying reference coverage cpth = ex.message[len(fs):-1] for err in accessibility_errors: if err in ex.message: cpth = self.get_coverage_path(dataset_id) self.pause_ingestion(self.get_stream_id(dataset_id)) break else: log.critical("Unmatched error: %s", ex.message) raise
def run_qc(self, data_product, reference_designator, parameter, qc_mapping): ''' Determines which algorithm the parameter should run, then evaluates the QC ''' # We key off of the OOI Short Name # DATAPRD_ALGRTHM_QC dp_ident, alg, qc = parameter.ooi_short_name.split('_') if dp_ident not in qc_mapping: return # No input! input_name = qc_mapping[dp_ident] try: doc = self.container.object_store.read_doc(reference_designator) except NotFound: return # NO QC lookups found if dp_ident not in doc: log.critical("Data product %s not in doc", dp_ident) return # No data product of this listing in the RD's entry # Lookup table has the rows for the QC inputs lookup_table = doc[dp_ident] # An instance of the coverage is loaded if we need to run an algorithm dataset_id = self.get_dataset(data_product) coverage = self.get_coverage(dataset_id) if not coverage.num_timesteps: # No data = no qc coverage.close() return try: # Get the lookup table info then run if alg.lower() == 'glblrng': row = self.recent_row(lookup_table['global_range']) min_value = row['min_value'] max_value = row['max_value'] self.process_glblrng(coverage, parameter, input_name, min_value, max_value) elif alg.lower() == 'stuckvl': row = self.recent_row(lookup_table['stuck_value']) resolution = row['resolution'] N = row['consecutive_values'] self.process_stuck_value(coverage, parameter,input_name, resolution, N) elif alg.lower() == 'trndtst': row = self.recent_row(lookup_table['trend_test']) ord_n = row['polynomial_order'] nstd = row['standard_deviation'] self.process_trend_test(coverage, parameter, input_name, ord_n, nstd) elif alg.lower() == 'spketst': row = self.recent_row(lookup_table['spike_test']) acc = row['accuracy'] N = row['range_multiplier'] L = row['window_length'] self.process_spike_test(coverage, parameter, input_name, acc, N, L) elif alg.lower() == "gradtst": row = self.recent_row(lookup_table["gradient_test"]) ddatdx = row["ddatdx"] mindx = row["mindx"] startdat = row["startdat"] if isinstance(startdat, basestring) and not startdat: startdat = np.nan if isinstance(mindx, basestring) and not mindx: mindx = np.nan toldat = row["toldat"] self.process_gradient_test(coverage, parameter, input_name, ddatdx, mindx, startdat, toldat) elif alg.lower() == 'loclrng': pass except KeyError: # No lookup table self.set_error(coverage, parameter) finally: coverage.close()
def run_qc(self, data_product, reference_designator, parameter, qc_mapping): ''' Determines which algorithm the parameter should run, then evaluates the QC ''' # We key off of the OOI Short Name # DATAPRD_ALGRTHM_QC log.error("Running QC %s (%s)", data_product.name, parameter.name) dp_ident, alg, qc = parameter.ooi_short_name.split('_') log.error("Identifier: %s", dp_ident) log.error("Test: %s", alg) if dp_ident not in qc_mapping: return # No input! input_name = qc_mapping[dp_ident] try: doc = self.container.object_store.read_doc(reference_designator) except NotFound: return # NO QC lookups found if dp_ident not in doc[reference_designator]: log.critical("Data product %s not in doc", dp_ident) return # No data product of this listing in the RD's entry # Lookup table has the rows for the QC inputs lookup_table = doc[reference_designator][dp_ident] log.error("lookup table found") # An instance of the coverage is loaded if we need to run an algorithm dataset_id = self.get_dataset(data_product) coverage = self.get_coverage(dataset_id) if not coverage.num_timesteps: # No data = no qc coverage.close() return try: # Get the lookup table info then run if alg.lower() == 'glblrng': row = self.recent_row(lookup_table['global_range']) min_value = row['min_value'] max_value = row['max_value'] self.process_glblrng(coverage, parameter, input_name, min_value, max_value) elif alg.lower() == 'stuckvl': log.error("Running Stuck Value") row = self.recent_row(lookup_table['stuck_value']) resolution = row['resolution'] N = row['consecutive_values'] self.process_stuck_value(coverage, parameter,input_name, resolution, N) elif alg.lower() == 'trndtst': log.error("Running Trend Test") row = self.recent_row(lookup_table['trend_test']) ord_n = row['polynomial_order'] nstd = row['standard_deviation'] self.process_trend_test(coverage, parameter, input_name, ord_n, nstd) elif alg.lower() == 'spketst': log.error("Runnign Spike Test") row = self.recent_row(lookup_table['spike_test']) acc = row['accuracy'] N = row['range_multiplier'] L = row['window_length'] self.process_spike_test(coverage, parameter, input_name, acc, N, L) finally: coverage.close()
def run_qc(self, data_product, reference_designator, parameter, qc_mapping, parameters): ''' Determines which algorithm the parameter should run, then evaluates the QC data_product - Data Product Resource reference_designator - reference designator string parameter - parameter context resource qc_mapping - a dictionary of { data_product_name : parameter_name } ''' # We key off of the OOI Short Name # DATAPRD_ALGRTHM_QC dp_ident, alg, qc = parameter.ooi_short_name.split('_') if dp_ident not in qc_mapping: return # No input! input_name = self.calibrated_candidates(data_product, parameter, qc_mapping, parameters) try: doc = self.container.object_store.read_doc(reference_designator) except NotFound: return # NO QC lookups found if dp_ident not in doc: log.critical("Data product %s not in doc", dp_ident) return # No data product of this listing in the RD's entry # Lookup table has the rows for the QC inputs lookup_table = doc[dp_ident] # An instance of the coverage is loaded if we need to run an algorithm dataset_id = self.get_dataset(data_product) coverage = self.get_coverage(dataset_id) if not coverage.num_timesteps: # No data = no qc coverage.close() return try: # Get the lookup table info then run if alg.lower() == 'glblrng': row = self.recent_row(lookup_table['global_range']) min_value = row['min_value'] max_value = row['max_value'] self.process_glblrng(coverage, parameter, input_name, min_value, max_value) elif alg.lower() == 'stuckvl': row = self.recent_row(lookup_table['stuck_value']) resolution = row['resolution'] N = row['consecutive_values'] self.process_stuck_value(coverage, parameter, input_name, resolution, N) elif alg.lower() == 'trndtst': row = self.recent_row(lookup_table['trend_test']) ord_n = row['polynomial_order'] nstd = row['standard_deviation'] self.process_trend_test(coverage, parameter, input_name, ord_n, nstd) elif alg.lower() == 'spketst': row = self.recent_row(lookup_table['spike_test']) acc = row['accuracy'] N = row['range_multiplier'] L = row['window_length'] self.process_spike_test(coverage, parameter, input_name, acc, N, L) elif alg.lower() == "gradtst": row = self.recent_row(lookup_table["gradient_test"]) ddatdx = row["ddatdx"] mindx = row["mindx"] startdat = row["startdat"] if isinstance(startdat, basestring) and not startdat: startdat = np.nan if isinstance(mindx, basestring) and not mindx: mindx = np.nan toldat = row["toldat"] self.process_gradient_test(coverage, parameter, input_name, ddatdx, mindx, startdat, toldat) elif alg.lower() == 'loclrng': row = self.recent_row(lookup_table["local_range"]) table = row['table'] dims = [] datlimz = [] for key in table.iterkeys(): # Skip the datlims if 'datlim' in key: continue dims.append(key) datlimz.append(table[key]) datlimz = np.column_stack(datlimz) datlim = np.column_stack([table['datlim1'], table['datlim2']]) self.process_local_range_test(coverage, parameter, input_name, datlim, datlimz, dims) except KeyError: # No lookup table self.set_error(coverage, parameter) finally: coverage.close()