def setup_module():
testing.init_blank_db_for_testing()
generator = tangos.testing.simulation_generator.SimulationGeneratorForTests(
)
ts1 = generator.add_timestep()
ts1_h1, ts2_h2 = generator.add_objects_to_timestep(2)
ts1_h1['dummy_property_1'] = np.arange(0, 100.0)
angmom = np.empty((100, 3))
angmom[:, :] = np.arange(0, 100.0).reshape((100, 1))
ts1_h1['dummy_property_2'] = angmom
ts1_h1['dummy_property_3'] = -2.5
ts1_h1_bh1 = tangos.core.halo.BH(ts1, 1)
tangos.get_default_session().add(ts1_h1_bh1)
ts1_h1_bh1["BH_mass"] = 1000.0
ts1_h1_bh2 = tangos.core.halo.BH(ts1, 2)
tangos.get_default_session().add(ts1_h1_bh2)
ts1_h1_bh2["BH_mass"] = 900.0
ts1_h1["BH"] = ts1_h1_bh1, ts1_h1_bh2
generator.add_timestep()
generator.add_objects_to_timestep(1)
generator.link_last_halos()
generator.add_timestep() # intentionally empty final timestep
def test_no_duplication(fresh_database):
run_writer_with_args("dummy_property")
assert db.get_default_session().query(db.core.HaloProperty).count()==15
run_writer_with_args("dummy_property") # should not create duplicates
assert db.get_default_session().query(db.core.HaloProperty).count() == 15
run_writer_with_args("dummy_property", "--force") # should create duplicates
assert db.get_default_session().query(db.core.HaloProperty).count() == 30
def test_for_loop_is_not_run_twice():
"""This test checks for an issue where if the number of CPUs exceeded the number of jobs for a task, the
entire task could be run twice"""
tangos.get_halo(1)['test_count'] = 0
tangos.get_default_session().commit()
pt.launch(_test_not_run_twice, 5)
assert tangos.get_halo(1)['test_count'] == 3
def test_no_duplication():
init_blank_simulation()
run_writer_with_args("dummy_property")
assert db.get_default_session().query(db.core.HaloProperty).count()==15
run_writer_with_args("dummy_property") # should not create duplicates
assert db.get_default_session().query(db.core.HaloProperty).count() == 15
run_writer_with_args("dummy_property", "--force") # should create duplicates
assert db.get_default_session().query(db.core.HaloProperty).count() == 30
def test_link_dependency(fresh_database):
run_writer_with_args("dummy_property_requiring_link")
assert db.get_default_session().query(db.core.HaloProperty).count() == 0
run_writer_with_args("dummy_link")
run_writer_with_args("dummy_property_requiring_link")
assert db.get_default_session().query(db.core.HaloProperty).count() == 15
def test_link_dependency():
init_blank_simulation()
run_writer_with_args("dummy_property_requiring_link")
assert db.get_default_session().query(db.core.HaloProperty).count() == 0
run_writer_with_args("dummy_link")
run_writer_with_args("dummy_property_requiring_link")
assert db.get_default_session().query(db.core.HaloProperty).count() == 15
def test_import():
existing_session = tangos.get_default_session()
testing.init_blank_db_for_testing(testing_db_name='imported_db_test')
new_session = tangos.get_default_session()
manager._db_import_export(new_session, existing_session)
differ = diff.TangosDbDiff(existing_session, new_session)
differ.compare()
assert not differ.failed, "Copied database differs; see log for details"
def test_unresolvable_proxy_object():
incomplete = po.IncompleteProxyObjectFromFinderId(10, 'halo')
assert incomplete.relative_to_timestep_id(1).resolve(
tangos.get_default_session()) is None
cache = toc.TimestepObjectCache(tangos.get_timestep('sim/ts1'))
assert incomplete.relative_to_timestep_cache(cache).resolve(
tangos.get_default_session()) is None
def _test_not_run_twice():
import time
# For this test we want a staggered start
time.sleep(pt.backend.rank() * 0.05)
for i in pt.distributed(list(range(3))):
with pt.ExclusiveLock("lock"):
tangos.get_halo(1)['test_count'] += 1
tangos.get_default_session().commit()
def test_proxy_object_from_id_and_timestep():
incomplete = po.IncompleteProxyObjectFromFinderId(1, 'halo')
assert incomplete.relative_to_timestep_id(1).resolve(
tangos.get_default_session()) == tangos.get_object('sim/ts1/halo_1')
assert incomplete.relative_to_timestep_id(2).resolve(
tangos.get_default_session()) == tangos.get_object('sim/ts2/halo_1')
incomplete = po.IncompleteProxyObjectFromFinderId(1, 'BH')
assert incomplete.relative_to_timestep_id(1).resolve(
tangos.get_default_session()) == tangos.get_object('sim/ts1/BH_1')
def test_missing_or_broken_class():
ts = tangos.get_timestep("sim/ts1")
for (i,h) in enumerate(ts.halos):
h["noclass"] = 10.0*i
h["brokenproperty"] = 10.0*i
tangos.get_default_session().commit()
noclass, = ts.calculate_all("noclass")
npt.assert_allclose(noclass, [0., 10., 20., 30.])
with warnings.catch_warnings(record=True) as w:
brokenclass, = ts.calculate_all("brokenproperty")
npt.assert_allclose(noclass, [0., 10., 20., 30.])
assert len(w)>0
def test_proxy_object_from_id_and_timestep_cache():
ts1_halo1 = tangos.get_object('sim/ts1/halo_1')
ts1_bh2 = tangos.get_object('sim/ts1/BH_2')
with tangos.testing.SqlExecutionTracker() as ctr:
incomplete = po.IncompleteProxyObjectFromFinderId(1, 'halo')
cache = toc.TimestepObjectCache(tangos.get_timestep('sim/ts1'))
assert incomplete.relative_to_timestep_cache(cache).resolve(
tangos.get_default_session()) == ts1_halo1
incomplete = po.IncompleteProxyObjectFromFinderId(2, 'BH')
assert incomplete.relative_to_timestep_cache(cache).resolve(
tangos.get_default_session()) == ts1_bh2
assert ctr.count_statements_containing("SELECT halos") == 1
def test_link_repr():
h1 = db.get_halo('dummy_sim_1/step.1/1')
h2 = db.get_halo('dummy_sim_1/step.1/2')
d_test = db.core.get_or_create_dictionary_item(db.get_default_session(),
"test")
l_obj = link.HaloLink(h1, h2, d_test, 1.0)
db.get_default_session().add(l_obj)
assert repr(
l_obj
) == "<HaloLink test dummy_sim_1/step.1/halo_1 to dummy_sim_1/step.1/halo_2 weight=1.00>"
l_obj = link.HaloLink(h1, h2, d_test, None)
db.get_default_session().add(l_obj)
assert repr(
l_obj
) == "<HaloLink test dummy_sim_1/step.1/halo_1 to dummy_sim_1/step.1/halo_2 weight=None>"
def setup():
testing.init_blank_db_for_testing()
creator = tangos.testing.simulation_generator.TestSimulationGenerator()
halo_offset = 0
for ts in range(1,4):
creator.add_timestep()
creator.add_objects_to_timestep(4)
creator.link_last_halos()
tangos.get_default_session().commit()
tangos.get_item("sim/ts1/halo_1")['test_image'] = np.zeros((500,500,3))
for ts in tangos.get_simulation(1).timesteps:
for h in ts.halos:
h['test_value'] = 1.0
tangos.get_default_session().commit()
creator.add_timestep() # add ts 4
creator.add_objects_to_timestep(3)
tangos.get_default_session().commit()
# now make the object have halo numbers which have a different ordering to the database IDs
tangos.get_item("sim/ts4/1")['test_value'] = 1.0
tangos.get_item("sim/ts4/2")['test_value'] = 2.0
tangos.get_item("sim/ts4/3")['test_value'] = 3.0
tangos.get_item("sim/ts4/1").halo_number = 10
tangos.get_default_session().commit()
creator = tangos.testing.simulation_generator.TestSimulationGenerator("simname/has/slashes")
creator.add_timestep()
creator.add_objects_to_timestep(1)
creator.add_timestep()
creator.add_objects_to_timestep(1)
creator.link_last_halos()
tangos.get_simulation(2).timesteps[0].halos[0]['test_value'] = 2.0
tangos.get_default_session().commit()
global app
app = TestApp(tangos.web.main({}))
def test_insert_properties():
for h in ts1.halos:
db.get_default_session().delete(
h) # remove previous objects so that we can add them afresh
adder = add_simulation.SimulationAdderUpdater(sim.get_output_handler())
adder.add_objects_to_timestep(ts1)
importer = property_importer.PropertyImporter()
importer.parse_command_line(
"Mvir Rvir hostHalo childHalo --for test_stat_files".split())
print(importer.options)
importer.run_calculation_loop()
npt.assert_almost_equal(ts1.halos[0]["Rvir"], 195.87)
npt.assert_almost_equal(ts1.halos[0]["Mvir"], 5.02432e+11)
with npt.assert_raises(KeyError):
ts1.halos[0]['hostHalo']
with npt.assert_raises(KeyError):
ts1.halos[1]['childHalo']
assert ts1.halos[2]['hostHalo'] == ts1.halos[0]
assert ts1.halos[3]['hostHalo'] == ts1.halos[0]
testing.assert_halolists_equal(ts1.halos[0]['childHalo'],
[ts1.halos[2], ts1.halos[3]])
def setup():
testing.init_blank_db_for_testing()
creator = tangos.testing.simulation_generator.TestSimulationGenerator()
halo_offset = 0
for ts in range(1, 4):
creator.add_timestep()
creator.add_objects_to_timestep(4)
creator.link_last_halos()
tangos.get_default_session().commit()
tangos.get_item("sim/ts1/halo_1")['test_image'] = np.zeros((500, 500, 3))
for ts in tangos.get_simulation(1).timesteps:
for h in ts.halos:
h['test_value'] = 1.0
tangos.get_default_session().commit()
creator = tangos.testing.simulation_generator.TestSimulationGenerator(
"simname/has/slashes")
creator.add_timestep()
creator.add_objects_to_timestep(1)
creator.add_timestep()
creator.add_objects_to_timestep(1)
creator.link_last_halos()
tangos.get_simulation(2).timesteps[0].halos[0]['test_value'] = 2.0
tangos.get_default_session().commit()
global app
app = TestApp(tangos.web.main({}))
def store_ids(self, ts, id_to_tree_id):
session = db.get_default_session()
objs = self.create_timestep_halo_dictionary(ts)
dict_obj = get_or_create_dictionary_item(session,
"consistent_trees_id")
props = []
for o in objs.values():
tree_id = id_to_tree_id.get(o.finder_id, None)
if tree_id is not None:
props.append(HaloProperty(o, dict_obj, tree_id))
session.add_all(props)
session.commit()
logger.info("%d consistent tree IDs added to step %s", len(props), ts)
def create_links(self, ts, ts_next, link_dictionary):
session = db.get_default_session()
d_id = get_or_create_dictionary_item(session, "consistent_trees_link")
objs_this = self.create_timestep_halo_dictionary(ts)
objs_next = self.create_timestep_halo_dictionary(ts_next)
links = []
for this_id, (next_id, merger_ratio) in link_dictionary.items():
this_obj = objs_this.get(this_id, None)
next_obj = objs_next.get(next_id, None)
if this_obj is not None and next_obj is not None:
links.append(HaloLink(this_obj, next_obj, d_id, 1.0))
links.append(HaloLink(next_obj, this_obj, d_id, merger_ratio))
session.add_all(links)
session.commit()
logger.info("%d links created between %s and %s", len(links), ts,
ts_next)
def get_object(id, session=None):
"""Get an object from an ID or an identifying string
Optionally, use the specified session.
:rtype: Halo
"""
if session is None:
session = get_default_session()
if isinstance(id, str) or isinstance(id, six.text_type):
sim, ts, halo = id.split("/")
ts = get_timestep(sim + "/" + ts, session)
return ts[halo]
else:
return session.query(Halo).filter_by(id=int(id)).first()
def wrapped(*args, **kwargs) :
if not isinstance(args[0], Base) :
args = list(args)
if isinstance(args[0], int) :
item = get_default_session().query(cl).filter_by(id=args[0]).first()
else :
item = get_item(args[0])
if not isinstance(item, cl) :
if isinstance(item, Simulation) and cl is Halo:
print("Picking first timestep and first halo")
item = item.timesteps[0].halos[0]
else :
raise RuntimeError("Path points to wrong type of db object %r"%item)
args[0] = item
return f(*args,**kwargs)
def get_simulation(id, session=None):
if session is None:
session = get_default_session()
if isinstance(id, str) or isinstance(id, six.text_type):
assert "/" not in id
if "%" in id:
match_clause = Simulation.basename.like(id)
else:
match_clause = Simulation.basename == id
res = session.query(Simulation).filter(match_clause)
num = res.count()
if num == 0:
raise RuntimeError("No simulation matches %r" % id)
elif num > 1:
raise RuntimeError("Multiple (%d) matches for %r" % (num, id))
else:
return res.first()
else:
return session.query(Simulation).filter_by(id=int(id)).first()
def get_timestep(id, session=None, sim=None):
if session is None:
session = get_default_session()
if isinstance(id, str) or isinstance(id, six.text_type):
if sim is None:
sim, ts = id.split("/")
sim = get_simulation(sim, session)
else:
ts = id
res = session.query(TimeStep).filter(
and_(TimeStep.extension.like(ts), TimeStep.simulation_id == sim.id))
num = res.count()
if num == 0:
raise RuntimeError("No timestep matches for %r" % id)
elif num > 1:
raise RuntimeError("Multiple (%d) matches for timestep %r of simulation %r" % (
num, ts, sim))
else:
return res.first()
else:
return session.query(TimeStep).filter_by(id=int(id)).first()
def test_link_property(fresh_database):
run_writer_with_args("dummy_link")
assert db.get_default_session().query(db.core.HaloLink).count() == 15
db.testing.assert_halolists_equal([db.get_halo(2)['dummy_link']], [db.get_halo(1)])
def test_incomplete_proxy_object():
with assert_raises(po.ProxyResolutionException):
po.IncompleteProxyObjectFromFinderId(1, 'halo').resolve(
tangos.get_default_session())
def test_proxy_object_by_id():
assert po.ProxyObjectFromDatabaseId(1).resolve(
tangos.get_default_session()) == tangos.get_object(1)
def test_link_property():
init_blank_simulation()
run_writer_with_args("dummy_link")
assert db.get_default_session().query(db.core.HaloLink).count() == 15
db.testing.assert_halolists_equal([db.get_halo(2)['dummy_link']], [db.get_halo(1)])
def get_haloproperty(id):
return get_default_session().query(HaloProperty).filter_by(id=id).first()
def all_creators():
return get_default_session().query(Creator).all()
def all_simulations(session=None):
if session is None:
session = get_default_session()
return session.query(Simulation).all()