def _feedback(self, inputs, outputs, initials=None, latches=None, keep_outputs=False): # TODO: remove in next version bump and put into wire. if latches is None: latches = inputs def blast(bmap, vals): return fn.lmapcat(bmap.get, vals) lmap = BundleMap( {l: self.imap[i].size for i, l in zip(inputs, latches)} ) if initials is not None: l2init = dict(self.aig.latch2init) l2init.update( {l: v for l, v in zip(latches, initials) if v is not None} ) initials = fn.lcat(l2init[l] for l in latches) aig = rebundle_aig(self.aig.feedback( inputs=blast(self.imap, inputs), outputs=blast(self.omap, outputs), latches=blast(lmap, latches), keep_outputs=keep_outputs, initials=initials, )) return aig
def load_stage(cls, dvcfile, name, stage_data, lock_data=None): assert all([name, dvcfile, dvcfile.repo, dvcfile.path]) assert stage_data and isinstance(stage_data, dict) path, wdir = resolve_paths(dvcfile.path, stage_data.get(Stage.PARAM_WDIR)) stage = loads_from(PipelineStage, dvcfile.repo, path, wdir, stage_data) stage.name = name deps = project(stage_data, [stage.PARAM_DEPS, stage.PARAM_PARAMS]) fill_stage_dependencies(stage, **deps) outs = project( stage_data, [stage.PARAM_OUTS, stage.PARAM_METRICS, stage.PARAM_PLOTS], ) stage.outs = lcat( output.load_from_pipeline(stage, data, typ=key) for key, data in outs.items()) if lock_data: stage.cmd_changed = lock_data.get(Stage.PARAM_CMD) != stage.cmd cls.fill_from_lock(stage, lock_data) return stage
def chain_nodes(nodes, chain): """Returns nodes matched by chain.""" if not chain: return nodes link, *rest = chain if link.func is Ops.const: return [] elif link.func is Ops.multi: coll, = link.args if isinstance(coll, list): coll = dict(enumerate(coll)) return { k: chain_nodes(nodes, subchain + rest) for k, subchain in coll.items() } elif link.func is notnone_fn: return ldistinct( lcat( chain_nodes(nodes, subchain + rest) for subchain in link.args)) else: # Doing this manually in case the link encapsulates a chain we can unpack func = link.func if link.args is None else link.func(*link.args) if isinstance(func, Chain): return chain_nodes(nodes, func + rest) else: next_value = func(nodes) return chain_nodes(next_value, rest) if is_elements(next_value) else nodes
def _dnf(where): """ Constructs DNF of where tree consisting of terms in form: (alias, attribute, value, negation) meaning `alias.attribute = value` or `not alias.attribute = value` if negation is False Any conditions other then eq are dropped. """ if isinstance(where, Lookup): # If where.lhs don't refer to a field then don't bother if not hasattr(where.lhs, 'target'): return SOME_TREE # Don't bother with complex right hand side either if isinstance(where.rhs, (QuerySet, Query, BaseExpression)): return SOME_TREE # Skip conditions on non-serialized fields if where.lhs.target not in serializable_fields( where.lhs.target.model): return SOME_TREE attname = where.lhs.target.attname if isinstance(where, Exact): return [[(where.lhs.alias, attname, where.rhs, True)]] elif isinstance(where, IsNull): return [[(where.lhs.alias, attname, None, where.rhs)]] elif isinstance(where, In) and len( where.rhs) < settings.CACHEOPS_LONG_DISJUNCTION: return [[(where.lhs.alias, attname, v, True)] for v in where.rhs] else: return SOME_TREE elif isinstance(where, NothingNode): return [] elif isinstance(where, (ExtraWhere, SubqueryConstraint, Exists)): return SOME_TREE elif len(where) == 0: return [[]] else: chilren_dnfs = lmap(_dnf, where.children) if len(chilren_dnfs) == 0: return [[]] elif len(chilren_dnfs) == 1: result = chilren_dnfs[0] else: # Just unite children joined with OR if where.connector == OR: result = lcat(chilren_dnfs) # Use Cartesian product to AND children else: result = lmap(lcat, product(*chilren_dnfs)) # Negating and expanding brackets if where.negated: result = [lmap(negate, p) for p in product(*result)] return result
def _fleiss_kappa(sample_sets): # If there is only one set then it can't be measured if len(sample_sets) == 1: return float('nan') all_samples_annos = lcat(sample_sets) categories = ldistinct(sv.annotation for sv in all_samples_annos) # If there is only one label then it can't be measured if len(categories) == 1: return float('nan') category_index = {c: i for i, c in enumerate(categories)} stats = defaultdict(lambda: [0] * len(categories)) for sv in all_samples_annos: stats[sv.sample_id][category_index[sv.annotation]] += 1 return fleiss_kappa(list(stats.values()))
def feedback(self, inputs, outputs, initials=None, latches=None, keep_outputs=False, signed=False): if latches is None: latches = inputs idrop, imap = fn.lsplit(lambda x: x[0] in inputs, self.input_map) odrop, omap = fn.lsplit(lambda x: x[0] in outputs, self.output_map) wordlens = [len(vals) for i, vals in idrop] new_latches = [(n, common.named_indexes(k, n)) for k, n in zip(wordlens, latches)] if initials is None: initials = [0 for _ in inputs] assert len(inputs) == len(outputs) == len(initials) == len(latches) initials = fn.lcat( common.encode_int(k, i, signed) for k, i in zip(wordlens, initials)) def get_names(key_vals): return fn.lcat(fn.pluck(1, key_vals)) aig = self.aig.feedback( inputs=get_names(idrop), outputs=get_names(odrop), latches=get_names(new_latches), initials=initials, keep_outputs=keep_outputs, ) imap, odrop, omap = map(frozenset, [imap, odrop, omap]) return AIGBV( aig=aig, input_map=imap, output_map=omap | (odrop if keep_outputs else frozenset()), latch_map=self.latch_map | set(new_latches), )
def discard(data): # just consider all ranges, ignoring the field association all_fields = lcat(data["fields"].values()) invalid_entries = set() invalid_field_sum = 0 for i, ticket in enumerate(data["nearby"]): for value in ticket: if not any(lo <= value <= hi for lo, hi in all_fields): invalid_field_sum += value invalid_entries.add(i) break # remove the invalid indexes in the nearby list for i in sorted(invalid_entries, reverse=True): del data["nearby"][i] return dict(invalid_field_sum=invalid_field_sum, nearby=data["nearby"])
def get_fields(graphql_schema, typename) -> Iterable[Tuple[str, str]]: IGNORE_FIELDS = ( ScalarTypeDefinitionNode, FragmentDefinitionNode, EnumTypeDefinitionNode, ) doc = parse_graphql_schema(graphql_schema) node: Node = find(doc.definitions, lambda x: x.name.value == typename) if getattr(node, "fields", None): return [(field.name.value, get_type_name(field.type)) for field in node.fields] else: if isinstance(node, (UnionTypeDefinitionNode,)): fields = [get_fields(graphql_schema, x.name.value) for x in node.types] fields = unique(lcat(fields), key=lambda x: x[0]) return fields elif isinstance(node, IGNORE_FIELDS): print(f"ignoring {node}") return [] else: raise Exception(f"unrecognized type for {node}")
def get_names(key_vals): return fn.lcat(fn.pluck(1, key_vals))
def eval_order(circ): return fn.lcat(toposort(_dependency_graph(circ.cones | circ.latch_cones)))
def containers(self): return lcat(pluck_attr("containers", self.replicasets.values()))
def sink(wordlen, inputs): blasted_inputs = [named_indexes(wordlen, i) for i in inputs] return aigbv.AIGBV( aig=aiger.sink(fn.lcat(blasted_inputs)), input_map=frozenset(fn.lzip(inputs, blasted_inputs)), )
def mget(keys): return lcat(redis_client.mget(chunk) for chunk in chunks(10000, keys))
def plugs(tile: np.ndarray) -> List[np.ndarray]: # the eight possible 1d borders for this tile # identified by the four borders and their flipped variants return lcat(edges(tile))
def get_mbta_info(cfgs): return pd.DataFrame.from_dict(f.lcat(map(get_mbta_station_info, cfgs)))
def eval_order(circ, *, concat: bool = True): """Return topologically sorted nodes in AIG.""" order = toposort(_dependency_graph(circ.cones | circ.latch_cones)) return fn.lcat(order) if concat else order