def test_multisource_across():
strategy = halo_finding.MultiSourceMultiHopStrategy(
tangos.get_items(["sim/ts2/1", "sim/ts2/2", "sim/ts2/3"]),
tangos.get_item("sim2"))
results = strategy.all()
testing.assert_halolists_equal(results, ["sim2/ts2/2", "sim2/ts2/1", None])
assert strategy._nhops_taken == 1
def test_earlier():
ts3_halos, ts1_halos = tangos.get_timestep("sim/ts3").calculate_all(
'dbid()', 'earlier(2).dbid()')
testing.assert_halolists_equal(
ts1_halos, ['sim/ts1/1', 'sim/ts1/2', 'sim/ts1/3', 'sim/ts1/1.1'])
testing.assert_halolists_equal(
ts3_halos, ['sim/ts3/1', 'sim/ts3/2', 'sim/ts3/3', 'sim/ts3/1.1'])
def test_multisource_with_nones_as_temptable():
strategy = halo_finding.MultiSourceMultiHopStrategy(
tangos.get_items(["sim/ts1/1", "sim/ts1/2", "sim/ts1/3", "sim/ts1/5"]),
tangos.get_item("sim/ts3"))
with strategy.temp_table() as table:
results = thl.all_halos_with_duplicates(table)
testing.assert_halolists_equal(results,["sim/ts3/1","sim/ts3/1","sim/ts3/2",None])
def test_major_progenitor_branch_from_merger_example():
import tangos.examples.mergers as mer
z_arr, ratio_arr, halo_arr = mer.get_mergers_of_major_progenitor(tangos.get_halo("sim/ts5/1"))
# the mergers example will see two major mergers happen - hard to avoid this without a lot of ad hoc rules
# Note also the major progenitor in the high-z collision is halo 2, not halo 1 (despite halo 1 being more massive).
# This is because most of the material in the remnant comes from halo 2.
testing.assert_halolists_equal(halo_arr, [("sim/ts4/1", "sim/ts4/2"), ("sim/ts2/2", "sim/ts2/1")])
def test_twostep_ordering():
strategy = halo_finding.MultiHopStrategy(tangos.get_item("sim/ts3/3"),
2,
'backwards',
order_by="time_asc")
all = strategy.all()
assert tangos.get_item("sim/ts1/4") == all[0]
assert tangos.get_item("sim/ts2/4") == all[1]
strategy = halo_finding.MultiHopStrategy(tangos.get_item("sim/ts3/3"),
2,
'backwards',
order_by="time_desc")
all = strategy.all()
assert tangos.get_item("sim/ts2/4") == all[0]
assert tangos.get_item("sim/ts1/4") == all[1]
strategy = halo_finding.MultiHopStrategy(tangos.get_item("sim/ts3/1"),
2,
'backwards',
order_by=["time_asc", "weight"])
all, weights = strategy.all_and_weights()
I = tangos.get_item
testing.assert_halolists_equal(
all,
[
("sim/ts1/1"),
("sim/ts1/2"),
# ("sim/ts1/1"), weaker route should NOT be returned by default
("sim/ts2/1"),
("sim/ts2/2")
])
def test_ahf_merger_tree_import():
importer = tools.ahf_merger_tree_importer.AHFTreeImporter()
importer.parse_command_line("--for test_ahf_merger_tree".split())
with log.LogCapturer():
importer.run_calculation_loop()
assert (np.array([
x.finder_offset for x in tangos.get_timestep(
"test_ahf_merger_tree/tiny_000832").halos.all()
]) == [1, 2, 3, 4, 5, 6, 7, 8, 9]).all()
testing.assert_halolists_equal(
tangos.get_timestep("test_ahf_merger_tree/tiny.000832").calculate_all(
"earlier(1)", object_typetag='halo')[0], [
"test_ahf_merger_tree/tiny.000640/halo_1",
"test_ahf_merger_tree/tiny.000640/halo_2",
"test_ahf_merger_tree/tiny.000640/halo_3",
"test_ahf_merger_tree/tiny.000640/halo_4",
"test_ahf_merger_tree/tiny.000640/halo_5",
"test_ahf_merger_tree/tiny.000640/halo_6",
"test_ahf_merger_tree/tiny.000640/halo_1",
"test_ahf_merger_tree/tiny.000640/halo_2"
])
assert tangos.get_halo("%/%640/halo_7").next == tangos.get_halo(
"%/%832/halo_1")
assert tangos.get_halo("%/%832/halo_1").previous == tangos.get_halo(
"%/%640/halo_1")
def test_multisource_preserves_order():
strategy = halo_finding.MultiSourceMultiHopStrategy(
tangos.get_items(["sim/ts1/3", "sim/ts1/2", "sim/ts1/1", "sim/ts1/5"]),
tangos.get_item("sim/ts3"))
results = strategy.all()
testing.assert_halolists_equal(
results, ["sim/ts3/2", "sim/ts3/1", "sim/ts3/1", None])
def test_multisource_with_nones():
strategy = halo_finding.MultiSourceMultiHopStrategy(
tangos.get_items(["sim/ts1/1", "sim/ts1/2", "sim/ts1/3", "sim/ts1/5"]),
tangos.get_item("sim/ts3"))
results = strategy.all()
testing.assert_halolists_equal(results,["sim/ts3/1","sim/ts3/1","sim/ts3/2",None])
assert strategy._nhops_taken==2 # despite not finding a match for ts1/5, the strategy should halt after 2 steps
def test_find_merger():
strategy = halo_finding.MultiHopMostRecentMergerStrategy(tangos.get_item("sim/ts3/1"))
results = strategy.all()
testing.assert_halolists_equal(results, ["sim/ts2/1","sim/ts2/2"] )
strategy = halo_finding.MultiHopMostRecentMergerStrategy(tangos.get_item("sim/ts3/4"))
results = strategy.all()
testing.assert_halolists_equal(results, ["sim/ts1/6", "sim/ts1/7"])
def test_multisource_forwards():
sources = tangos.get_items(["sim/ts1/1","sim/ts1/2","sim/ts1/3","sim/ts1/4"])
results = halo_finding.MultiSourceMultiHopStrategy(sources,
tangos.get_item("sim/ts3")).all()
testing.assert_halolists_equal(results, ["sim/ts3/1", "sim/ts3/1", "sim/ts3/2", "sim/ts3/3"])
single_latest = [i.latest for i in sources]
testing.assert_halolists_equal(single_latest, ["sim/ts3/1", "sim/ts3/1", "sim/ts3/2", "sim/ts3/3"])
def test_earlier_equal_weight():
"""Regression test for problem where earlier(x) or later(x) failed when there was more than one link
with the identical same weight, meaning the major progenitor was undefined. An interim fix picks one or the
other possibility, so that at least the code does not crash. However, a better long-term solution should
be found - see TODO in multi_source.py"""
ts5_halo, ts4_halo = tangos.get_timestep("sim/ts5").calculate_all("dbid()","earlier(1).dbid()")
testing.assert_halolists_equal(ts5_halo, ['sim/ts5/1'])
assert testing.halolists_equal(ts4_halo, ['sim/ts4/1']) or testing.halolists_equal(ts4_halo, ['sim/ts4/2'])
def test_earlier_equal_weight():
"""Regression test for problem where earlier(x) or later(x) failed when there was more than one link
with the identical same weight, meaning the major progenitor was undefined. Now the match is
disambiguiated by looking for the link with the highest primary key (not very physical, but at least
unambiguous)."""
ts5_halo, ts4_halo = tangos.get_timestep("sim/ts5").calculate_all("dbid()","earlier(1).dbid()")
testing.assert_halolists_equal(ts5_halo, ['sim/ts5/1'])
assert testing.halolists_equal(ts4_halo, ['sim/ts4/1']) or testing.halolists_equal(ts4_halo, ['sim/ts4/2'])
def test_merging():
source, dest = tangos.get_timestep("sim/ts1").calculate_all("halo_number()","later(2)", object_typetag='halo')
assert np.all(source==[1,2,3,4,6,7])
testing.assert_halolists_equal(dest, ['sim/ts3/1', 'sim/ts3/1', 'sim/ts3/2', 'sim/ts3/3', 'sim/ts3/4', 'sim/ts3/4'])
source, dest = tangos.get_timestep("sim/ts1").calculate_all("halo_number()", "latest()", object_typetag='halo')
print(source,dest)
assert np.all(source==[1,2,3,4,5,6,7])
testing.assert_halolists_equal(dest, ['sim/ts3/1', 'sim/ts3/1', 'sim/ts3/2', 'sim/ts3/3', 'sim/ts1/5',
'sim/ts3/4', 'sim/ts3/4'])
def test_historical_value_finding():
vals = db.get_halo("sim/ts3/1").calculate_for_progenitors("testval")
halo = db.get_halo("sim/ts3/1")
assert_halolists_equal([halo.calculate("find_progenitor(testval, 'max')")], ["sim/ts2/1"])
assert_halolists_equal([halo.calculate("find_progenitor(testval, 'min')")], ["sim/ts3/1"])
timestep = db.get_timestep("sim/ts3")
assert_halolists_equal(timestep.calculate_all("find_progenitor(testval, 'max')")[0], ["sim/ts2/1", "sim/ts3/2"])
assert_halolists_equal(timestep.calculate_all("find_progenitor(testval, 'min')")[0], ["sim/ts3/1", "sim/ts1/3"])
assert_halolists_equal(db.get_timestep("sim/ts1").calculate_all("find_descendant(testval, 'min')")[0],
["sim/ts3/1", "sim/ts1/3", "sim/ts1/4", "sim/ts1/5"])
def test_offset_outputs_dont_confuse_match():
# This tests for a bug where crosslinked timesteps at slightly different times could confuse the
# search for a progentior or descendant because the recursive search strayed into a different simulation
tangos.get_timestep("sim2/ts2").time_gyr *= 1.01
try:
ts1_2_next = tangos.get_item("sim/ts2/2").next
ts1_2_later = tangos.get_item("sim/ts2/2").calculate("later(1)")
correct = ["sim/ts3/1"]
testing.assert_halolists_equal([ts1_2_next], correct)
testing.assert_halolists_equal([ts1_2_later], correct)
finally:
tangos.get_timestep("sim2/ts2").time_gyr /= 1.01
def test_single_match_from_multiple_routes():
"""Test that even if multiple possible matches for an object in another simulation exist,
only the highest weight result is returned"""
# multiple routes in time tracing calculation (didn't previously fail)
h = tangos.get_halo("sim3/ts3/1")
testing.assert_halolists_equal([h.calculate("match('sim3/ts1')")],
[tangos.get_halo("sim3/ts1/1")])
# multiple routes in tracing to another simulation (did previously fail)
# The multiple routes here are sim3/ts1/1 -> sim2/ts1/1, sim2/ts1/2 -> sim/ts1/1
h = tangos.get_halo("sim3/ts1/1")
testing.assert_halolists_equal([h.calculate("match('sim')")],
[tangos.get_halo("sim/ts1/1")])
def test_latest_earliest():
earliest = db.get_halo("sim/ts3/1").calculate("earliest()")
assert earliest == db.get_halo("sim/ts3/1").earliest
latest = db.get_halo("sim/ts1/2").calculate("latest()")
assert latest == db.get_halo("sim/ts1/2").latest
latest = db.get_halo("sim/ts3/1").calculate("latest()")
assert latest == db.get_halo("sim/ts3/1") # this is already the latest. See #123
earliest_no_ancestors = db.get_halo("sim/ts2/3").calculate("earliest()") # has no predecessor in ts1. See #124
assert earliest_no_ancestors == db.get_halo("sim/ts2/3")
multi_earliest, = db.get_timestep("sim/ts2").calculate_all("earliest()")
assert_halolists_equal(multi_earliest, ['sim/ts1/2', 'sim/ts1/3', 'sim/ts2/3'])
def test_consistent_tree_import():
importer = tools.consistent_trees_importer.ConsistentTreesImporter()
importer.parse_command_line(
"--for test_gadget_rockstar --with-ids".split())
with log.LogCapturer():
importer.run_calculation_loop()
assert (
tangos.get_timestep("test_gadget_rockstar/snapshot_014").calculate_all(
"consistent_trees_id",
object_typetag='halo')[0] == [17081, 19718, 19129]).all()
testing.assert_halolists_equal(
tangos.get_timestep("test_gadget_rockstar/snapshot_014").calculate_all(
"earlier(1)", object_typetag='halo')[0], [
"test_gadget_rockstar/snapshot_013/halo_2",
"test_gadget_rockstar/snapshot_013/halo_4"
])
assert tangos.get_halo("%/%13/halo_1").next == tangos.get_halo(
"%/%14/phantom_1")
def test_insert_properties():
for h in ts1.halos:
db.get_default_session().delete(
h) # remove previous objects so that we can add them afresh
adder = add_simulation.SimulationAdderUpdater(sim.get_output_handler())
adder.add_objects_to_timestep(ts1)
importer = property_importer.PropertyImporter()
importer.parse_command_line(
"Mvir Rvir hostHalo childHalo --for test_stat_files".split())
print(importer.options)
importer.run_calculation_loop()
npt.assert_almost_equal(ts1.halos[0]["Rvir"], 195.87)
npt.assert_almost_equal(ts1.halos[0]["Mvir"], 5.02432e+11)
with npt.assert_raises(KeyError):
ts1.halos[0]['hostHalo']
with npt.assert_raises(KeyError):
ts1.halos[1]['childHalo']
assert ts1.halos[2]['hostHalo'] == ts1.halos[0]
assert ts1.halos[3]['hostHalo'] == ts1.halos[0]
testing.assert_halolists_equal(ts1.halos[0]['childHalo'],
[ts1.halos[2], ts1.halos[3]])
def test_major_progenitor_from_minor_progenitor():
generator = tangos.testing.simulation_generator.TestSimulationGenerator(
"sim3")
ts1 = generator.add_timestep()
generator.add_objects_to_timestep(4)
ts2 = generator.add_timestep()
generator.add_objects_to_timestep(3)
generator.link_last_halos_using_mapping({
1: 2,
2: 1,
3: 3,
4: 1
},
adjust_masses=True)
# ts1->ts2: most massive and second most massive halos swap rank ordering by mass because of the
# merger with ts1/h4.
ts3 = generator.add_timestep()
generator.add_objects_to_timestep(2)
# ts2->ts3: there is a major merger of the most massive halos (ts2/h1+ts2/h2)->ts3/h1
generator.link_last_halos_using_mapping({
1: 1,
2: 1,
3: 2
},
adjust_masses=True)
# Check major progenitor correctly reported one step back by MultiSourceMultiHopStrategy
progen_in_ts2 = halo_finding.MultiSourceMultiHopStrategy(
tangos.get_items(["sim3/ts3/1"]), tangos.get_item("sim3/ts2")).all()
testing.assert_halolists_equal(progen_in_ts2, ['sim3/ts2/1'])
# Check major progenitor correctly reported two steps back by MultiHopMajorProgenitorsStrategy
all_progenitors, weights = halo_finding.MultiHopMajorProgenitorsStrategy(
tangos.get_item("sim3/ts3/1")).all_and_weights()
testing.assert_halolists_equal(all_progenitors,
['sim3/ts2/1', 'sim3/ts1/2'])
# Check major progenitor correctly reported two steps back by MultiSourceMultiHopStrategy
# This is where a failure occurred in the past -- the behaviour was inequivalent to always choosing
# the highest weight branch (which is what MultiHopMajorProgenitorsStrategy does).
#
# In the example constructed above, the mapping from ts3 halo 1 reaches:
# ts1, h4 (weight 0.26)
# ts1, h2 (weight 0.35)
# ts1, h1 (weight 0.39)
#
# It looks from these weights like we ought to be picking ts1/h1.
#
# However the correct major progenitor is h2, because one identifies a major progenitor at each step.
# In step ts2, h1 has weight 0.61 and h2 has weight 0.39. So ts2/h1 is the major progenitor. And then,
# going back to ts3, ts1/h2 (weight 0.57) is the major progenitor to ts2/h1.
progen_in_ts1 = halo_finding.MultiSourceMultiHopStrategy(
tangos.get_items(["sim3/ts3/1"]), tangos.get_item("sim3/ts1")).all()
testing.assert_halolists_equal(progen_in_ts1, ['sim3/ts1/2'])
def test_calculate_for_progenitors_bh():
h = tangos.get_halo("sim/ts3/bh_1")
objs, = h.calculate_for_progenitors("dbid()")
testing.assert_halolists_equal(objs, ['sim/ts3/BH_1', 'sim/ts2/BH_1', 'sim/ts1/BH_1'])
def test_historical_value_finding_missing_data():
sources, targets = db.get_timestep("sim/ts3").calculate_all("path()", "find_progenitor(testvalpartial, 'max')")
assert_halolists_equal(sources, ["sim/ts3/1", "sim/ts3/2"])
assert_halolists_equal(targets, ["sim/ts1/2", "sim/ts1/3"])
def test_major_descendants():
results = halo_finding.MultiHopMajorDescendantsStrategy(
tangos.get_item("sim/ts1/2"), include_startpoint=True).all()
testing.assert_halolists_equal(results,
["sim/ts1/2", "sim/ts2/1", "sim/ts3/1"])
def test_across():
results = halo_finding.MultiHopStrategy(tangos.get_item("sim/ts2/2"),
directed='across').all()
testing.assert_halolists_equal(results, ["sim2/ts2/1"])
def test_multisource_with_duplicates():
results = halo_finding.MultiSourceMultiHopStrategy(
tangos.get_items(["sim/ts1/1", "sim/ts1/2", "sim/ts1/3"]),
tangos.get_item("sim/ts3")).all()
testing.assert_halolists_equal(results,
["sim/ts3/1", "sim/ts3/1", "sim/ts3/2"])