def invalidated_update(self, **kwargs): clone = self._clone().nocache() clone._for_write = True # affects routing objects = list(clone) rows = clone.update(**kwargs) # TODO: do not refetch objects but update with kwargs in simple cases? pks = {obj.pk for obj in objects} for obj in chain(objects, self.model.objects.filter(pk__in=pks)): invalidate_obj(obj) return rows
def invalidated_update(self, **kwargs): clone = self._clone().nocache() clone._for_write = True # affects routing objects = list(clone) rows = clone.update(**kwargs) # TODO: do not refetch objects but update with kwargs in simple cases? pks = {obj.pk for obj in objects} for obj in chain(objects, self.model.objects.filter(pk__in=pks)): invalidate_obj(obj) return rows
def _cohens_kappa(annos1, annos2): assert set(s.sample_id for s in annos1) == set(s.sample_id for s in annos2) categories = distinct(sv.annotation or '' for sv in chain(annos1, annos2)) category_index = {c: i for i, c in enumerate(categories)} table = np.zeros((len(categories), len(categories))) annos1 = sorted(annos1, key=attrgetter('sample_id')) annos2 = sorted(annos2, key=attrgetter('sample_id')) for sv1, sv2 in zip(annos1, annos2): table[category_index[sv1.annotation or ''], category_index[sv2.annotation or '']] += 1 return cohens_kappa(table, return_results=False)
def invalidated_update(self, **kwargs): clone = self._clone().nocache() clone._for_write = True # affects routing objects = list(clone) rows = clone.update(**kwargs) # TODO: do not refetch objects but update with kwargs in simple cases? # We use clone database to fetch new states, as this is the db they were written to. # Using router with new_objects may fail, using self may return slave during lag. pks = {obj.pk for obj in objects} new_objects = self.model.objects.filter(pk__in=pks).using(clone.db) for obj in chain(objects, new_objects): invalidate_obj(obj, using=clone.db) return rows
def perms_as_stream(perms: Sequence[Sequence], l_pad=None, r_pad=None, pad_num=1): """ for a sequence of sequences return an item stream out of those sequences (like chain) padded by l_pad and r_pad: ex: for [[1,2], [3,4]] with default pads we would get: -> None, 1, 2, None, None, 3, 4, None this is useful when we want a context window over a collection, but dont want to miss the first / last chars as middle of the window """ left_pad = [l_pad] * pad_num right_pad = [r_pad] * pad_num return chain(*[left_pad + list(p) + right_pad for p in perms])
def invalidated_update(self, **kwargs): clone = self._clone().nocache() clone._for_write = True # affects routing objects = list(clone) rows = clone.update(**kwargs) # TODO: do not refetch objects but update with kwargs in simple cases? # We use clone database to fetch new states, as this is the db they were written to. # Using router with new_objects may fail, using self may return slave during lag. pks = {obj.pk for obj in objects} new_objects = self.model.objects.filter(pk__in=pks).using(clone.db) for obj in chain(objects, new_objects): invalidate_obj(obj, using=clone.db) return rows
def values(self, *fields, **expressions): """ Extended version supporting renames: .values('id', 'name', author__name='author') """ renames = select_values(isa(six.string_types), expressions) if not renames: return base.values(self, *fields, **expressions) elif django.VERSION >= (1, 11): rename_expressions = walk_values(F, renames) expressions.update(rename_expressions) return base.values(self, *fields, **expressions) else: f_to_name = flip(renames) rename = lambda d: {f_to_name.get(k, k): v for k, v in d.items()} return base.values(self, *chain(fields, f_to_name)).map(rename)
def encode_game(g, store, ce=None): # obj, *non_obj = relabel_specs(g, counter_examples) obj, *non_obj = g.specs obj = stl.utils.discretize(obj, dt=g.model.dt, distribute=True) non_obj = { stl.utils.discretize(phi, dt=g.model.dt, distribute=True) for phi in non_obj if phi != stl.TOP } # Constraints robustness = rob_encode.encode(obj, store, 0) # TODO dynamics = rob_encode.encode_dynamics(g, store) other = cat(bool_encode.encode(psi, store, 0) for psi in non_obj) return fn.chain(robustness, dynamics, other), obj
def encode_game(g, store, ce=None): # obj, *non_obj = relabel_specs(g, counter_examples) obj, *non_obj = g.specs obj = stl.utils.discretize(obj, dt=g.model.dt, distribute=True) non_obj = { stl.utils.discretize(phi, dt=g.model.dt, distribute=True) for phi in non_obj if phi != stl.TOP } # Constraints robustness = rob_encode.encode(obj, store, 0) # TODO dynamics = rob_encode.encode_dynamics(g, store) other = cat(bool_encode.encode(psi, store, 0) for psi in non_obj) return fn.chain(robustness, dynamics, other), obj
def _cohens_kappa(annos1, annos2): assert set(s.sample_id for s in annos1) == set(s.sample_id for s in annos2) categories = ldistinct(sv.annotation for sv in chain(annos1, annos2)) # If there is only one label then it can't be measured if len(categories) == 1: return float('nan') category_index = {c: i for i, c in enumerate(categories)} table = np.zeros((len(categories), len(categories))) annos1 = sorted(annos1, key=attrgetter('sample_id')) annos2 = sorted(annos2, key=attrgetter('sample_id')) for sv1, sv2 in zip(annos1, annos2): table[category_index[sv1.annotation], category_index[sv2.annotation]] += 1 return cohens_kappa(table, return_results=False)
def primparams(fn: Callable) -> Union[Singular, Tuple[Any]]: "Aproximate type signature of function `fn´ in Python primitive types" fullparams = params(fn) # type: ignore def getparams(): return filter(lambda p: p.default is inspect.Signature.empty, fullparams.values()) positional = map(attrgetter('annotation'), filter(ispos, getparams())) keyword = map(attrgetter('annotation'), filter(iskw, getparams())) kind = attrgetter('kind') variadic = tuple(... for x in fullparams.values() if kind(x) == Parameter.VAR_POSITIONAL) + tuple( Mapping for x in fullparams.values() if kind(x) == Parameter.VAR_KEYWORD) return unbox(tuple(fy.chain(positional, keyword, variadic)))
def check_schema(cls, schema: dict) -> None: msg_types_found = { k: False for k in funcy.chain(settings.HEDWIG_MESSAGE_ROUTING, settings.HEDWIG_CALLBACKS) } # custom validation for Hedwig - TODO ideally should just be represented in json-schema file as meta schema, # however jsonschema lib doesn't support draft 06 which is what's really required here errors = [] if not schema.get('schemas'): errors.append( "Invalid schema file: expected key 'schemas' with non-empty value" ) else: for msg_type, versions in schema['schemas'].items(): if not isinstance(versions, dict) or not versions: errors.append( f"Invalid definition for message type: '{msg_type}', value must contain a dict of " f"valid versions") else: for major_version, definition in versions.items(): try: if (msg_type, int(major_version)) in msg_types_found: msg_types_found[(msg_type, int(major_version))] = True except ValueError: errors.append( f"Invalid version '{major_version}' for message type: '{msg_type}'" ) if not isinstance(definition, dict) and not definition: errors.append( f"Invalid version '{major_version}' for message type: '{msg_type}'" ) for (msg_type, major_version), found in msg_types_found.items(): if not found: errors.append( f"Schema not found for '{msg_type}' v{major_version}") if errors: raise SchemaError(errors)
def _encode(self, prev_latch, action, state): (step, lmap), circ1 = self._cutlatches() curr_step = step << aiger.source(prev_latch) for a, v in action.items(): size = circ1.imap[a].size const = aiger_bv.source(size, aiger_bv.decode_int(v, signed=False), name=a, signed=False) curr_step <<= const.aig expr = uatom(1, "##valid") == 1 for k, v in fn.chain(state.items()): expr &= _constraint(k, v) curr_step >>= expr.aig query = curr_step >> aiger.sink(prev_latch.keys()) assert len(query.outputs) == 1 model = solve(query) assert model is not None # Fill in any model don't cares. model = fn.merge({i: False for i in circ1.aig.inputs}, model) # HACK. Put model back into bitvector. coins = circ1.imap.omit(self.inputs).unblast(model) if len(prev_latch) > 0: next_latch_circ = curr_step >> aiger.sink(expr.aig.outputs) next_latch = next_latch_circ(model)[0] assert next_latch.keys() == prev_latch.keys() prev_latch = next_latch return coins, prev_latch
def gen_directed_dists(part1, part2): """Generator for directed hausdorff distances.""" # TODO: remimplement using sortedcontainers approx1, approx2 = {part1.tree}, {part2.tree} new1, new2 = approx1, approx2 dists = {} imin = mdtr.Interval(-float('inf'), float('inf')) while True: # Compute dists. comparisons = fn.chain(product(new1, approx2), product(approx1, new2)) dists.update({(n1, n2): node_dist(n1, n2) for n1, n2 in comparisons}) n1, n2 = best_worst_case(approx1, approx2, dists) imin = dists[n1, n2] yield imin # Best score must lie in the interval. # Refine. if not n1.view().is_point: new1 = set(n1.children) approx1 = (approx1 | new1) - {n1} if not n2.view().is_point: new2 = set(n2.children) approx2 = (approx2 | new2) - {n2}
def calc_validation_stats(serie_validation_pk, recalc=False): serie_validation = SerieValidation.objects.select_for_update().get( pk=serie_validation_pk) # Guard from double update, so that user stats won't be messed up if not recalc and serie_validation.samples_total is not None: return series_tag = serie_validation.series_tag if not series_tag: return # Compare to annotation sample_validations = serie_validation.sample_validations.all() sample_annotations = series_tag.sample_tags.all() if set(r.sample_id for r in sample_validations) \ != set(r.sample_id for r in sample_annotations): logger.error("Sample sets mismatch for validation %d" % serie_validation_pk) # It's either bug when making annotation or samples set really changed series_tag.is_active = False series_tag.save() # TODO: notify annotation author to redo it return _fill_concordancy(sample_validations, sample_annotations) # Fill serie validation stats serie_validation.samples_total = len(sample_validations) serie_validation.samples_concordant = sum(s.concordant for s in sample_validations) serie_validation.annotation_kappa = _cohens_kappa(sample_validations, sample_annotations) # Compare to other validations earlier_validations = series_tag.validations.filter(pk__lt=serie_validation_pk, ignored=False) \ .order_by('pk') # TODO: use .prefetch_related() earlier_sample_validations = group_by( lambda v: v.serie_validation_id, SampleValidation.objects.filter( serie_validation__in=earlier_validations)) if not serie_validation.concordant: serie_validation.agrees_with = first( v for v in earlier_validations if v.created_by_id != serie_validation.created_by_id and is_samples_concordant(earlier_sample_validations[v.pk], sample_validations)) # NOTE: this includes kappas against your prev validations serie_validation.best_kappa = max( chain([serie_validation.annotation_kappa], (_cohens_kappa(sample_validations, sv) for sv in earlier_sample_validations.values()))) serie_validation.save() # Calculate fleiss kappa for all existing annotations/validations annotation_sets = [sample_annotations, sample_validations] \ + earlier_sample_validations.values() series_tag.fleiss_kappa = _fleiss_kappa(annotation_sets) if not serie_validation.on_demand and not serie_validation.ignored \ and (serie_validation.concordant or serie_validation.agrees_with): series_tag.agreed = earlier_validations.count() + 1 series_tag.save() # TODO: make this separate task ? if not recalc and not serie_validation.on_demand and not serie_validation.by_incompetent: _update_user_stats(serie_validation) # including payment ones # TODO: make this separate task ? # Reschedule validation if no agreement found if not series_tag.agreed and not recalc and not serie_validation.on_demand \ and not serie_validation.by_incompetent: # Schedule revalidations with priority < 0, that's what new validations have, # to phase out garbage earlier _reschedule_validation(serie_validation, priority=series_tag.fleiss_kappa - 1)