def _scan_transpose(ct, consts, init, xs, forward, length, jaxpr): assert consts is None and init is None assert type(xs) is tuple a, res = xs assert a is None and res is not None # jaxpr :: d -> c -> (a, res) -> (c, b) # jaxpr_lifted :: res -> (d, c, a) -> (c, b) # jaxpr_lifted_trans :: res -> (CT c, CT b) -> (CT d, CT c, CT a) # jaxpr_trans :: * -> (CT c, CT d) -> (CT b, res) -> ((CT c, CT d), CT a) assert type(jaxpr.jaxpr.invars[2]) is tuple # assume restructuring jaxpr_lifted = rearrange_binders( lambda d, c, a_res: (a_res[1], (d, c, a_res[0])), jaxpr) jaxpr_lifted_trans = _transpose_jaxpr(jaxpr_lifted) jaxpr_trans = _move_stuff_and_add_add(jaxpr_lifted_trans) c_aval, b_aval = jaxpr.out_aval d_aval, c_aval2, _ = jaxpr.in_avals assert c_aval == c_aval2 bs_aval = _promote_aval_rank(length, b_aval) ct_d = ad_util.zeros_like_aval(d_aval) ct_c, ct_bs = ad.instantiate_zeros_aval(core.AbstractTuple((c_aval, bs_aval)), ct) carry_ct = core.pack((ct_c, ct_d)) # jaxpr_trans :: * -> (CT c, CT d) -> (CT b, res) -> ((CT c, CT d), CT a) core.check_jaxpr(jaxpr_trans.jaxpr) unit_aval, (ct_c_aval, ct_d_aval), (ct_b_aval, _) = jaxpr_trans.in_avals assert core.lattice_join(ct_c_aval, core.get_aval(ct_c)) == ct_c_aval assert core.lattice_join(ct_d_aval, core.get_aval(ct_d)) == ct_d_aval out = scan_p.bind( core.unit, carry_ct, core.pack((ct_bs, res)), forward=not forward, length=length, jaxpr=jaxpr_trans) (ct_init, ct_consts), ct_as = out return ct_consts, ct_init, (ct_as, None)
def _instantiate_zeros(arg, tan): """Turn special ad.zero tangents into arrays of 0s.""" if type(tan) is not ad.Zero: return tan try: aval = arg.aval return ad.instantiate_zeros_aval(aval, tan) except (AttributeError, KeyError): # We get here for regular Python values return ad.zeros_like_jaxval(arg)
def transposed(res, b_bar): _, (_, a_bar) = ad.backward_pass(jaxpr.jaxpr, jaxpr.literals, (), (res, None), b_bar) a_bar = ad.instantiate_zeros_aval(jaxpr.in_avals[1], a_bar) return a_bar