def compute_obsels(self, computed_trace, from_scratch=False): """I implement :meth:`.interface.IMethod.compute_obsels`. """ diag = Diagnosis("sparql.compute_obsels") source = computed_trace.source_traces[0] parameters = computed_trace.parameters_as_dict parameters["__destination__"] = computed_trace.uri parameters["__source__"] = source.uri scope = parameters.get('scope', 'trace') try: if scope == 'store': data = ConjunctiveGraph(source.service.store) elif scope == 'base': data = PrefixConjunctiveView(source.base.uri, source.service.store) else: # scope == 'trace' data = source.obsel_collection.get_state({"refresh": "no"}) sparql = parameters["sparql"] % parameters result = data.query(sparql, base=source.obsel_collection.uri).graph replace_obsels(computed_trace, result, ("inherit" in parameters)) except Exception, exc: LOG.warn(traceback.format_exc()) diag.append(unicode(exc))
def compute_trace_description(self, computed_trace): """I implement :meth:`.interface.IMethod.compute_trace_description`. """ diag = Diagnosis("compute_trace_description for <{}>".format(self.uri)) cstate = { 'errors': None, 'log_mon_tag': None, 'pse_mon_tag': None, 'str_mon_tag': None, 'custom': {}, } params = self._prepare_params(computed_trace, diag) if len(computed_trace.source_traces) != 1: diag.append("Method <{}> expects exactly one source".format(self.uri)) params = None if params is not None: model = self.compute_model(computed_trace, params, diag) origin = self.compute_origin(computed_trace, params, diag) with computed_trace.edit(_trust=True) as editable: editable.add((computed_trace.uri, KTBS.hasModel, model)) editable.add((computed_trace.uri, KTBS.hasOrigin, origin)) self.init_state(computed_trace, params, cstate['custom'], diag) if not diag: cstate["errors"] = list(diag) computed_trace.metadata.set((computed_trace.uri, METADATA.computation_state, Literal(json_dumps(cstate)) )) return diag
def compute_trace_description(self, computed_trace): """I implement :meth:`.interface.IMethod.compute_trace_description`. """ diag = Diagnosis("sparql.compute_trace_description") src, params = self._prepare_source_and_params(computed_trace, diag) if src is not None: if params.get('scope') == 'store': config = computed_trace.service.config if not config.has_section('sparql') \ or not config.has_option('sparql', 'allow-scope-ktbs') \ or not config.getboolean('sparql', 'allow-scope-ktbs'): diag.append( "Scope 'ktbs' must be explicitly allowed in config, " "as it raises privacy issued.") assert params is not None model = params.get("model") if model is None: model = src.model_uri else: model = URIRef(model) origin = Literal(params.get("origin") or src.origin) with computed_trace.edit(_trust=True) as editable: editable.add((computed_trace.uri, KTBS.hasModel, model)) editable.add((computed_trace.uri, KTBS.hasOrigin, origin)) return diag
def compute_obsels(self, computed_trace, from_scratch=False): """I implement :meth:`.interface.IMethod.compute_obsels`. """ diag = Diagnosis("pipe.compute_trace_description") eff_src_uri = computed_trace.metadata.value(computed_trace.uri, METADATA.effective_source) effective_source = computed_trace.factory(eff_src_uri) if effective_source is None: msg = "Effective source can not be found <%s>" % eff_src_uri LOG.error(msg) diag.append(msg) return diag else: return _FUSION_IMPL.compute_obsels(computed_trace, from_scratch, [effective_source], diag)
def force_state_refresh(self, parameters=None): """I override `~rdfrest.cores.ICore.force_state_refresh`:meth: I recompute my data if needed. """ if self.__forcing_state_refresh: return self.__forcing_state_refresh = True try: super(ComputedTrace, self).force_state_refresh(parameters) for src in self._iter_effective_source_traces(): src.force_state_refresh(parameters) if self.metadata.value(self.uri, METADATA.dirty, None) is not None: # we *first* unset the dirty bit, so that recursive calls to # get_state do not result in an infinite recursion self.metadata.remove((self.uri, METADATA.dirty, None)) with self.edit(_trust=True) as editable: editable.remove((self.uri, KTBS.hasDiagnosis, None)) editable.remove((self.uri, KTBS.hasModel, None)) editable.remove((self.uri, KTBS.hasOrigin, None)) try: diag = self._method_impl.compute_trace_description( self) except BaseException, ex: LOG.warn(traceback.format_exc()) diag = Diagnosis( "exception raised while computing trace description", [ex.message]) if not diag: editable.add((self.uri, KTBS.hasDiagnosis, Literal(unicode(diag)))) for ttr in self.iter_transformed_traces(): ttr._mark_dirty() finally: del self.__forcing_state_refresh
def compute_obsels(self, computed_trace, from_scratch=False): """I implement :meth:`.interface.IMethod.compute_obsels`. """ diag = Diagnosis("paralel.compute_trace_description") eff_src_uris = computed_trace.metadata.objects( computed_trace.uri, METADATA.effective_source) effective_sources = [] for uri in eff_src_uris: eff_src = computed_trace.factory(uri) if eff_src is None: msg = "Effective source can not be found <%s>" % eff_src_uri LOG.warning(msg) diag.append("WARN: " + msg) else: effective_sources.append(eff_src) return _FUSION_IMPL.compute_obsels(computed_trace, from_scratch, effective_sources, diag)
def compute_obsels(self, computed_trace, from_scratch=False): """I implement :meth:`.interface.IMethod.compute_obsels`. """ diag = Diagnosis("paralel.compute_trace_description") eff_src_uris = computed_trace.metadata.objects(computed_trace.uri, METADATA.effective_source) effective_sources = [] for uri in eff_src_uris: eff_src = computed_trace.factory(uri) if eff_src is None: msg = "Effective source can not be found <%s>" % eff_src_uri LOG.warning(msg) diag.append("WARN: "+msg) else: effective_sources.append(eff_src) return _FUSION_IMPL.compute_obsels(computed_trace, from_scratch, effective_sources, diag)
class _FakeMethod(IMethod): """I mimic IMethod when no matching method is found. """ def __init__(self, uri): # IMethod.__init__ is not called #pylint: disable=W0231 self.uri = uri self.diag = Diagnosis("_FakeMethod") self.diag.append("%s is unreachable and unimplemented; can not compute trace" % uri) def compute_trace_description(self, computed_trace): """I implement :meth:`..method.interface.IMethod.compute_trace_description`. """ return self.diag def compute_obsels(self, computed_trace, from_scratch=False): """I implement :meth:`..method.interface.IMethod.compute_obsels`. """ return self.diag
class _FakeMethod(IMethod): """I mimic IMethod when no matching method is found. """ def __init__(self, uri): # IMethod.__init__ is not called #pylint: disable=W0231 self.uri = uri self.diag = Diagnosis("_FakeMethod") self.diag.append( "%s is unreachable and unimplemented; can not compute trace" % uri) def compute_trace_description(self, computed_trace): """I implement :meth:`..method.interface.IMethod.compute_trace_description`. """ return self.diag def compute_obsels(self, computed_trace, from_scratch=False): """I implement :meth:`..method.interface.IMethod.compute_obsels`. """ return self.diag
def check_new_graph(cls, service, uri, parameters, new_graph, resource=None, added=None, removed=None): """I implement :meth:`ILocalCore.check_new_graph`. I check what the mixins can not check. """ # unused arguments #pylint: disable=W0613 # Do NOT call super method, as this is the base implementation. diag = Diagnosis("check_new_graph") begin = new_graph.value(uri, KTBS.hasBegin) end = new_graph.value(uri, KTBS.hasEnd) try: begin = int(begin) end = int(begin) if end < begin: diag.append("End timestamp is before begin timestamp [%s,%s]" % (begin, end)) except ValueError: LOG.info(traceback.format_exc()) diag.append("Can not convert timestamps to int: [%s,%s]" % (begin, end)) # TODO SOON check that graph only contains a bounded description # of the obsel return diag
def compute_trace_description(self, computed_trace, _sources=None): """I implement :meth:`.interface.IMethod.compute_trace_description`. """ diag = Diagnosis("fusion.compute_trace_description") sources = _sources or computed_trace.source_traces params = computed_trace.parameters_as_dict critical = False if len(sources) < 1: diag.append("Method ktbs:fusion expects at least one source") critical = True critical, fusion_params = \ self._get_fusion_parameters(params, diag, critical) for key in params: diag.append("WARN: Parameter %s is not used by ktbs:fusion" % key) if not critical: self._do_compute_trace_description(computed_trace, sources, fusion_params, diag) self._init_cstate(computed_trace, diag) return diag
def compute_obsels(self, computed_trace, from_scratch=False): """I implement :meth:`.interface.IMethod.compute_obsels`. """ diag = Diagnosis("compute_obsels for <{}>".format(self.uri)) cstate = json_loads( computed_trace.metadata.value(computed_trace.uri, METADATA.computation_state)) if from_scratch: cstate["log_mon_tag"] = None cstate["pse_mon_tag"] = None cstate["str_mon_tag"] = None errors = cstate.get("errors") if errors: for i in errors: diag.append(i) return diag source_obsels = computed_trace.source_traces[0].obsel_collection from logging import getLogger; LOG = getLogger() if cstate["str_mon_tag"] == source_obsels.str_mon_tag: monotonicity = STRICT_MON elif cstate["pse_mon_tag"] == source_obsels.pse_mon_tag: monotonicity = PSEUDO_MON elif cstate["log_mon_tag"] == source_obsels.log_mon_tag: monotonicity = LOGIC_MON else: monotonicity = NOT_MON self.do_compute_obsels(computed_trace, cstate['custom'], monotonicity, diag) cstate["log_mon_tag"] = source_obsels.log_mon_tag cstate["pse_mon_tag"] = source_obsels.pse_mon_tag cstate["str_mon_tag"] = source_obsels.str_mon_tag computed_trace.metadata.set((computed_trace.uri, METADATA.computation_state, Literal(json_dumps(cstate)) )) return diag
def compute_obsels(self, computed_trace, from_scratch=False): """I implement :meth:`.interface.IMethod.compute_obsels`. """ diag = Diagnosis("sparql.compute_obsels") source = computed_trace.source_traces[0] parameters = computed_trace.parameters_as_dict parameters["__destination__"] = computed_trace.uri parameters["__source__"] = source.uri scope = parameters.get('scope', 'trace') try: if scope == 'base': data = PrefixConjunctiveView(source.base.uri, source.service.store) else: # scope == 'trace' data = source.obsel_collection.get_state({"refresh":"no"}) sparql = parameters["sparql"] % parameters result = data.query(sparql, base=source.obsel_collection.uri).graph replace_obsels(computed_trace, result, ("inherit" in parameters)) except Exception, exc: LOG.warn(traceback.format_exc()) diag.append(unicode(exc))
def force_state_refresh(self, parameters=None): """I override `~rdfrest.cores.ICore.force_state_refresh`:meth: I recompute the obsels if needed. """ refresh_param = (_REFRESH_VALUES[parameters.get("refresh")] if parameters else 1) if refresh_param == 0 or self.__forcing_state_refresh: return with self.lock(self): self.__forcing_state_refresh = True try: LOG.debug("forcing state refresh <%s>", self.uri) super(ComputedTraceObsels, self).force_state_refresh(parameters) trace = self.trace if refresh_param == 2: parameters['refresh'] = 'default' # do not transmit 'force' to sources for src in trace._iter_effective_source_traces(): src.obsel_collection.force_state_refresh(parameters) if (refresh_param >= 2 or self.metadata.value(self.uri, METADATA.dirty, None) is not None): with self.service: # start transaction if not already started LOG.info("recomputing <%s>", self.uri) # we *first* unset the dirty bit, so that recursive calls to # get_state do not result in an infinite recursion self.metadata.remove((self.uri, METADATA.dirty, None)) trace.force_state_refresh() impl = trace._method_impl # friend #pylint: disable=W0212 try: diag = impl.compute_obsels(trace, refresh_param >= 2) except BaseException, ex: LOG.warn(traceback.format_exc()) diag = Diagnosis( "exception raised while computing obsels", [ex.message], sys.exc_traceback, ) if not diag: self.metadata.set((self.uri, METADATA.dirty, Literal("yes"))) raise CanNotProceedError, unicode(diag), diag.traceback finally: del self.__forcing_state_refresh
def compute_trace_description(self, computed_trace): """I implement :meth:`.interface.IMethod.compute_trace_description`. """ diag = Diagnosis("sparql.compute_trace_description") src, params = self._prepare_source_and_params(computed_trace, diag) if src is not None: assert params is not None model = params.get("model") if model is None: model = src.model_uri else: model = URIRef(model) origin = Literal(params.get("origin") or src.origin) with computed_trace.edit(_trust=True) as editable: editable.add((computed_trace.uri, KTBS.hasModel, model)) editable.add((computed_trace.uri, KTBS.hasOrigin, origin)) return diag
def compute_trace_description(self, computed_trace): """I implement :meth:`.interface.IMethod.compute_trace_description`. """ diag = Diagnosis("fusion.compute_trace_description") cstate = { "method": "fusion", "last_seens": {}, "old_log_mon_tags": {}, } srcs, params = self._prepare_source_and_params(computed_trace, diag) if srcs is not None: assert params is not None model = params.get("model") if model is not None: model = URIRef(model) else: models = set(src.model_uri for src in srcs) if len(models) > 1: diag.append("Sources have different models and no target " "model is explicitly specified") else: model = models.pop() origin = params.get("origin") if origin is None: origins = set(src.origin for src in srcs) if len(origins) > 1: diag.append("Sources have different origins and no target " "origin is explicitly specified") else: origin = origins.pop() origin = Literal(origin) with computed_trace.edit(_trust=True) as editable: if model: editable.add((computed_trace.uri, KTBS.hasModel, model)) if origin: editable.add((computed_trace.uri, KTBS.hasOrigin, origin)) if not diag: cstate["errors"] = list(diag) computed_trace.metadata.set( (computed_trace.uri, METADATA.computation_state, Literal(json_dumps(cstate)))) return diag
def __init__(self, uri): # IMethod.__init__ is not called #pylint: disable=W0231 self.uri = uri self.diag = Diagnosis("_FakeMethod") self.diag.append("%s is unreachable and unimplemented; can not compute trace" % uri)
def test_default_creation(self): diag = Diagnosis() assert diag eq_(str(diag), "diagnosis: ok")
def test_append(self): diag = Diagnosis() diag.append("error 1") assert not diag eq_(str(diag), "diagnosis: ko\n* error 1\n")
def compute_trace_description(self, computed_trace): """I implement :meth:`.interface.IMethod.compute_trace_description`. """ diag = Diagnosis("pipe.compute_trace_description") src, params = self._prepare_source_and_params(computed_trace, diag) _FUSION_IMPL._init_cstate(computed_trace, diag) if src is None: return diag method_params = params['method_params'] methods = params['methods'] source = computed_trace.source_traces[0] prev = source base = computed_trace.base id_template = '_%s_{}'.format(computed_trace.id) # create or update intermediate traces for i, method in enumerate(methods): int_trace_id = id_template % i int_trace = base.get(int_trace_id) if int_trace is None: int_trace = base.create_computed_trace(int_trace_id, method, method_params[i], [prev]) else: with int_trace.edit(_trust=True) as editable: int_trace_uri = int_trace.uri # we do not use the high-level API here, # because it would force the state to refresh # even if no actual changes are done editable.set((int_trace_uri, KTBS.hasMethod, method.uri)) editable.remove((int_trace_uri, KTBS.hasParameter, None)) for item in method_params[i].items(): editable.add((int_trace_uri, KTBS.hasParameter, Literal('{}={}'.format(*item)))) prev = int_trace effective_source = prev computed_trace.metadata.set( (computed_trace.uri, METADATA.effective_source, effective_source.uri)) # inherit trace description from effecive_source model_uri = effective_source.model_uri origin = effective_source.state.value(effective_source.uri, KTBS.hasOrigin) with computed_trace.edit(_trust=True) as editable: editable.set((computed_trace.uri, KTBS.hasModel, model_uri)) editable.set((computed_trace.uri, KTBS.hasOrigin, origin)) # remove spurious intermediate traces i = len(methods) to_del = [] while True: int_trace_id = id_template % i int_trace = base.get(int_trace_id) if int_trace is not None: to_del.append(int_trace) else: break i += 1 for int_trace in to_del[::-1]: int_trace.delete()
def test_with_title(self): diag = Diagnosis("foo") assert diag eq_(str(diag), "foo: ok")
def test_and(self): diag1 = Diagnosis("foo", ["error 1"]) diag2 = Diagnosis("bar", ["error 2"]) diag = diag1 & diag2 assert not diag assert str(diag) == "foo: ko\n* error 1\n* error 2\n"
def test_append(self): diag = Diagnosis() diag.append("error 1") assert not diag assert str(diag) == "diagnosis: ko\n* error 1\n"
def compute_trace_description(self, computed_trace): """I implement :meth:`.interface.IMethod.compute_trace_description`. """ diag = Diagnosis("parallel.compute_trace_description") params = computed_trace.parameters_as_dict critical = False if len(computed_trace.source_traces) != 1: msg = "Method ktbs:parallel expects exactly one source" LOG.error(msg) diag.append(msg) critical = True critical, fusion_params = \ _FUSION_IMPL._get_fusion_parameters(params, diag, critical) if "methods" not in params: msg = "Method ktbs:parallel requires parameter methods" LOG.error(msg) diag.append(msg) critical = True else: method_uris = [ i for i in params.pop('methods').split(' ') if i != '' ] if len(method_uris) < 1: msg = "Method ktbs:parellel must have at least one method" LOG.error(msg) diag.append(msg) critical = True methods = [] method_params = [] for uri in method_uris: m = factory(uri) if len(method_uris) < 1: msg = "Sub-method <{}> is not accessible".format(uri) LOG.warning(msg) diag.append("WARN: " + msg) else: methods.append(m) method_params.append({}) parallel_params = { 'methods_uris': method_uris, 'methods': methods, 'method_params': method_params, } if len(params) > 1: diag.append("Method ktbs:parallel does not support " "additional parameters yet") # TODO implement a way to dispatch parameters to submethods if not critical: sources = self._prepare_intermediate_traces( computed_trace, parallel_params) _FUSION_IMPL._do_compute_trace_description( computed_trace, sources, fusion_params, diag) _FUSION_IMPL._init_cstate(computed_trace, diag)
def __init__(self, uri): # IMethod.__init__ is not called #pylint: disable=W0231 self.uri = uri self.diag = Diagnosis("_FakeMethod") self.diag.append( "%s is unreachable and unimplemented; can not compute trace" % uri)
def test_default_creation(self): diag = Diagnosis() assert diag assert str(diag) == "diagnosis: ok"
def compute_obsels(self, computed_trace, from_scratch=False, _sources=None, _diag=None): """I implement :meth:`.interface.IMethod.compute_obsels`. """ diag = _diag if _diag is not None else Diagnosis( "fusion.compute_obsels") cstate = json_loads( computed_trace.metadata.value(computed_trace.uri, METADATA.computation_state)) if from_scratch: for key in ("last_seens", "old_log_mon_tags"): cstate[key] = {} errors = cstate.get("errors") if errors: for i in errors: diag.append(i) return diag effective_sources = _sources or computed_trace.source_traces # start anew if sources have changed or have been modified in a # non-monotonic way old_log_mon_tags = cstate["old_log_mon_tags"] target_obsels = computed_trace.obsel_collection for src in effective_sources: old_tag = old_log_mon_tags.get(src.uri) if old_tag != src.obsel_collection.log_mon_tag: target_obsels._empty() # friend #pylint: disable=W0212 LOG.debug("non-monotonic %s", computed_trace) cstate["last_seens"] = {} cstate["old_log_mon_tags"] = old_log_mon_tags = {} break if not old_log_mon_tags: cstate["old_log_mon_tags"] = old_log_mon_tags = dict( (src.uri, src.obsel_collection.log_mon_tag) for src in effective_sources) last_seens = cstate["last_seens"] target_uri = computed_trace.uri with target_obsels.edit(_trust=True) as editable: target_contains = editable.__contains__ target_add = editable.add for src in effective_sources: src_uri = src.uri src_triples = src.obsel_collection.get_state({ "refresh": "no" }).triples for obs in src.iter_obsels(begin=last_seens.get(src_uri), refresh="no"): last_seens[src_uri] = obs.begin new_obs_uri = translate_node(obs.uri, computed_trace, src_uri, True) if target_contains( (new_obs_uri, KTBS.hasTrace, target_uri)): LOG.debug("--- skipping %s", new_obs_uri) continue # already added LOG.debug("--- keeping %s", obs) target_add((new_obs_uri, KTBS.hasTrace, target_uri)) target_add((new_obs_uri, KTBS.hasSourceObsel, obs.uri)) for _, pred, obj in src_triples((obs.uri, None, None)): if pred == KTBS.hasTrace \ or pred == KTBS.hasSourceObsel: continue new_obj = translate_node(obj, computed_trace, src_uri, True) target_add((new_obs_uri, pred, new_obj)) for subj, pred, _ in src_triples((None, None, obs.uri)): if pred == KTBS.hasTrace \ or pred == KTBS.hasSourceObsel: continue new_subj = translate_node(subj, computed_trace, src_uri, True) target_add((new_subj, pred, new_obs_uri)) computed_trace.metadata.set( (computed_trace.uri, METADATA.computation_state, Literal(json_dumps(cstate)))) return diag
def test_with_title(self): diag = Diagnosis("foo") assert diag assert str(diag) == "foo: ok"
def test_with_values(self): diag = Diagnosis("foo", ["error 1", "error 2"]) assert not diag assert str(diag) == "foo: ko\n* error 1\n* error 2\n"
def compute_trace_description(self, computed_trace): """I implement :meth:`.interface.IMethod.compute_trace_description`. """ diag = Diagnosis("parallel.compute_trace_description") params = computed_trace.parameters_as_dict critical = False if len(computed_trace.source_traces) != 1: msg = "Method ktbs:parallel expects exactly one source" LOG.error(msg) diag.append(msg) critical = True critical, fusion_params = \ _FUSION_IMPL._get_fusion_parameters(params, diag, critical) if "methods" not in params: msg = "Method ktbs:parallel requires parameter methods" LOG.error(msg) diag.append(msg) critical = True else: method_uris = [i for i in params.pop('methods').split(' ') if i != ''] if len(method_uris) < 1: msg = "Method ktbs:parellel must have at least one method" LOG.error(msg) diag.append(msg) critical = True methods = [] method_params = [] for uri in method_uris: m = factory(uri) if len(method_uris) < 1: msg = "Sub-method <{}> is not accessible".format(uri) LOG.warning(msg) diag.append("WARN: "+msg) else: methods.append(m) method_params.append({}) parallel_params = { 'methods_uris': method_uris, 'methods': methods, 'method_params': method_params, } if len(params) > 1: diag.append("Method ktbs:parallel does not support " "additional parameters yet") # TODO implement a way to dispatch parameters to submethods if not critical: sources = self._prepare_intermediate_traces(computed_trace, parallel_params) _FUSION_IMPL._do_compute_trace_description(computed_trace, sources, fusion_params, diag) _FUSION_IMPL._init_cstate(computed_trace, diag)