def setUp(self):
super(SubmitLightConeTests, self).setUp()
GlobalParameterFactory.create(parameter_name='maximum-random-light-cones', parameter_value='10')
GlobalParameterFactory(parameter_name='INITIAL_JOB_STATUS', parameter_value='HELD')
GlobalParameterFactory(parameter_name='job_too_large_warning', parameter_value='JOB_WARNING')
simulation = SimulationFactory.create(box_size=500)
galaxy_model = GalaxyModelFactory.create()
dataset = DataSetFactory.create(simulation=simulation, galaxy_model=galaxy_model, max_job_box_count=15)
self.redshifts = ['1.23456789', '2.987654321', '3.69154927', '4.567890123']
for redshift in self.redshifts:
SnapshotFactory.create(dataset=dataset, redshift=redshift)
DataSetPropertyFactory.create(dataset=dataset)
StellarModelFactory.create()
BandPassFilterFactory.create()
self.survey_preset = SurveyPresetFactory.create(name='Preset 1', parameters='<xml></xml>')
self.username = "******"
password = "******"
self.user = UserFactory.create(username=self.username, password=password)
self.parameters = """<lightcone>
<database_type>sqlite</database_type>
<database_name>random.db</database_name>
<catalogue_geometry>cone</catalogue_geometry>
</lightcone>
"""
self.login(self.username, password)
self.visit('mock_galaxy_factory')
self.click('tao-tabs-' + 'light_cone')
def setUp(self):
super(MockImageTests, self).setUp()
GlobalParameterFactory.create(parameter_name='maximum-random-light-cones', parameter_value='10')
GlobalParameterFactory(parameter_name='INITIAL_JOB_STATUS', parameter_value='HELD')
GlobalParameterFactory(parameter_name='job_too_large_warning', parameter_value='JOB_WARNING')
simulation = SimulationFactory.create(box_size=500)
galaxy_model = GalaxyModelFactory.create()
dataset = DataSetFactory.create(simulation=simulation, galaxy_model=galaxy_model, max_job_box_count=15)
DustModelFactory.create()
self.redshifts = ['1.23456789', '2.987654321', '3.69154927', '4.567890123']
for redshift in self.redshifts:
SnapshotFactory.create(dataset=dataset, redshift=redshift)
DataSetPropertyFactory.create(dataset=dataset)
StellarModelFactory.create()
BandPassFilterFactory.create()
self.survey_preset = SurveyPresetFactory.create(name='Preset 1', parameters='<xml></xml>')
self.username = "******"
password = "******"
self.user = UserFactory.create(username=self.username, password=password, is_staff=True, is_superuser=True)
self.parameters = """<lightcone>
<database_type>sqlite</database_type>
<database_name>random.db</database_name>
<catalogue_geometry>cone</catalogue_geometry>
</lightcone>
"""
self.login(self.username, password)
self.visit('mock_galaxy_factory')
self.click('tao-tabs-' + 'light_cone')
## fill in ligght-cone and sed (correctly)
self.select(self.lc_id('catalogue_geometry'), 'Light-Cone')
self.fill_in_fields({
'ra_opening_angle': '2',
'dec_opening_angle': '2',
'redshift_min': '1',
'redshift_max': '2',
}, id_wrap=self.lc_id)
self.click(self.lc_2select('op_add_all'))
self.click('tao-tabs-sed')
self.click(self.sed('apply_sed'))
self.click(self.sed_2select('op_add_all'))
self.click('tao-tabs-record_filter')
self.fill_in_fields({
'min': '1\n',
'max': '10\n',
}, id_wrap=self.rf_id)
self.click('tao-tabs-mock_image')
def setUp(self):
super(DiskStorageTests, self).setUp()
username = '******'
password = '******'
self.user = UserFactory.create(username=username, is_superuser=True)
self.user.set_password(password)
self.user.save()
self.login(username, password)
output_path = 'job_output'
self.job = JobFactory.create(user=self.user, status=Job.COMPLETED, output_path=output_path)
file_content = 'abc' * 2000000
file_name_to_content = {'file_name': file_content}
helper.create_file(os.path.join(settings.FILES_BASE, output_path), 'file_name', file_name_to_content)
self.job.save()
self.simulation = SimulationFactory.create()
self.galaxy = GalaxyModelFactory.create()
self.dataset = DataSetFactory.create(simulation=self.simulation, galaxy_model=self.galaxy)
self.output_prop = DataSetPropertyFactory.create(dataset=self.dataset, name='Central op', is_filter=False)
self.filter = DataSetPropertyFactory.create(name='CentralMvir rf', units="Msun/h", dataset=self.dataset)
self.computed_filter = DataSetPropertyFactory.create(name='Computed Property', dataset=self.dataset, is_computed=True)
self.snapshot = SnapshotFactory.create(dataset=self.dataset, redshift='0.33')
self.survey_preset = SurveyPresetFactory.create(name='Preset 1', parameters='<xml></xml>')
self.default_disk_quota = GlobalParameterFactory.create(parameter_name='default_disk_quota', parameter_value='6')
def setUp(self):
super(JobTypeFormTests, self).setUp()
GlobalParameterFactory.create(parameter_name='maximum-random-light-cones', parameter_value='10')
box_sim = SimulationFactory.create(box_size=500, name='simulation_000')
lc_sim = SimulationFactory.create(box_size=60,name='simulation_001')
self.params_path = os.path.join(PROJECT_DIR, 'test_data', 'params.xml')
params_string = open(self.params_path).read()
for i in range(3):
g = GalaxyModelFactory.create(name='galaxy_model_%03d' % i)
ds = DataSetFactory.create(simulation=box_sim, galaxy_model=g, max_job_box_count=25)
for j in range(10):
SnapshotFactory.create(dataset=ds, redshift=str(j)+".0")
for j in range(3):
dsp = DataSetPropertyFactory.create(dataset=ds, label='parameter_%03d label' % j, name='name_%03d' % j, description='description_%03d' % j)
ds.default_filter_field = dsp
ds.save()
for i in range(4,8):
g = GalaxyModelFactory.create(name='galaxy_model_%03d' % i)
ds = DataSetFactory.create(simulation=lc_sim, galaxy_model=g, max_job_box_count=25, id=i)
for j in range(10):
SnapshotFactory.create(dataset=ds, redshift=str(j)+".0")
for j in range(4,7):
dsp = DataSetPropertyFactory.create(dataset=ds, label='parameter_%03d label' % j, name='name_%03d' % j, description='description_%03d' % j)
ds.default_filter_field = dsp
ds.save()
for i in range(3):
StellarModelFactory.create(label='stellar_label_%03d' % i,
name='model{0}/sspm.dat'.format(i),
description='<p>Description %d </p>' % i)
BandPassFilterFactory.create(label='Band pass filter %03d' % i, filter_id='%d' % i)
DustModelFactory.create(name='Dust_model_%03d.dat' % i, label='Dust model %03d' % i, details='<p>Detail %d </p>' % i)
SurveyPresetFactory.create(name='Preset %d' % i, parameters=params_string)
username = "******"
password = "******"
self.user = UserFactory.create(username=username, password=password, is_staff=True, is_active=True, is_superuser=True)
self.login(username, password)
self.visit('mock_galaxy_factory')
self.click('tao-tabs-job_type')
def setUp(self):
super(MockGalaxyFactoryTests, self).setUp()
OUTPUT_FORMATS = [
{'value':'csv', 'text':'CSV (Text2)', 'extension':'csv'},
{'value':'hdf5', 'text':'HDF5', 'extension':'hdf5'},
{'value': 'fits', 'text': 'FITS', 'extension': 'fits'},
{'value': 'votable', 'text': 'VOTable', 'extension': 'xml'}
]
self.output_formats = GlobalParameterFactory.create(parameter_name='output_formats', parameter_value=OUTPUT_FORMATS)
self.simulation = SimulationFactory.create()
galaxy_model = GalaxyModelFactory.create()
self.dataset = DataSetFactory.create(simulation=self.simulation, galaxy_model=galaxy_model, max_job_box_count=11)
self.filter = DataSetPropertyFactory.create(dataset=self.dataset)
self.filter_long = DataSetPropertyFactory.create(dataset=self.dataset, data_type=DataSetProperty.TYPE_LONG_LONG)
self.filter_float = DataSetPropertyFactory.create(dataset=self.dataset, data_type=DataSetProperty.TYPE_FLOAT)
self.computed_filter = DataSetPropertyFactory.create(dataset=self.dataset, is_computed=True)
self.dataset.default_filter_field = self.filter
self.dataset.save()
SnapshotFactory.create(dataset=self.dataset)
self.survey_preset = SurveyPresetFactory.create(name='Preset 1', parameters='<xml></xml>')
self.stellar_model = StellarModelFactory.create()
self.bandpass = BandPassFilterFactory.create()
self.user = UserFactory.create()
#expected_timestamp = "2012-11-13 13:45:32+1000"
time.frozen_time = datetime.datetime(2012, 11, 13, 13, 45, 32, 0, UtcPlusTen())
self.output_format = OUTPUT_FORMATS[0]['value']
self.default_form_values = {}
self.default_form_values['light_cone'] = {
'catalogue_geometry': LightConeForm.CONE,
'dark_matter_simulation': 1,
'galaxy_model': self.dataset.id,
'output_properties': [str(self.filter.id)],
'ra_opening_angle': '2',
'dec_opening_angle': '2',
'redshift_min': '1',
'redshift_max': '2',
'number_of_light_cones': '1',
}
self.default_form_values['sed'] = {
'apply_sed': False,
}
self.default_form_values['record_filter'] = {'filter' : 'X-'+NO_FILTER}
def setUp(self):
super(FilterTests, self).setUp()
simulation1 = SimulationFactory.create()
simulation2 = SimulationFactory.create()
self.redshifts = ['1.123456789', '2.123456789', '3.123456789']
for unused in range(4):
galaxy_model = GalaxyModelFactory.create()
dataset = DataSetFactory.create(simulation=simulation1, galaxy_model=galaxy_model, max_job_box_count=2)
DataSetPropertyFactory.create(dataset=dataset)
for redshift in self.redshifts:
SnapshotFactory.create(dataset=dataset, redshift=redshift)
for unused in range(5):
galaxy_model = GalaxyModelFactory.create()
dataset = DataSetFactory.create(simulation=simulation2, galaxy_model=galaxy_model)
DataSetPropertyFactory.create(dataset=dataset)
dsp = DataSetPropertyFactory.create(dataset=dataset, is_filter=False)
dataset.default_filter_field = dsp
dataset.save()
for redshift in self.redshifts:
SnapshotFactory.create(dataset=dataset, redshift=redshift)
self.bp_filters = []
for unused in range(3):
self.bp_filters.append(BandPassFilterFactory.create())
StellarModelFactory.create()
username = "******"
password = "******"
UserFactory.create(username=username, password=password)
self.login(username, password)
self.visit('mock_galaxy_factory')
self.click('tao-tabs-' + 'light_cone')
self.select(self.lc_id('catalogue_geometry'), 'Light-Cone')
self.select_dark_matter_simulation(simulation1)
self.select_galaxy_model(simulation1.galaxymodel_set.all().order_by('id')[0])
initial_simulation = Simulation.objects.all().order_by('id')[0]
initial_galaxy_model = initial_simulation.galaxymodel_set.all().order_by('id')[0]
self.initial_dataset = DataSet.objects.get(simulation=initial_simulation, galaxy_model=initial_galaxy_model)
GlobalParameterFactory(parameter_name='INITIAL_JOB_STATUS', parameter_value='HELD')
def setUp(self):
super(JobTest, self).setUp()
self.username = '******'
self.password = '******'
self.user = UserFactory.create(username=self.username, is_superuser=True)
self.user.set_password(self.password)
self.user.save()
# self.job_description = 'This is a job description'
self.job = JobFactory.create(user=self.user)
GlobalParameterFactory.create(parameter_name='maximum-random-light-cones', parameter_value='10000')
self.simulation = SimulationFactory.create()
self.galaxy = GalaxyModelFactory.create()
self.dataset = DataSetFactory.create(simulation=self.simulation, galaxy_model=self.galaxy)
self.output_prop = DataSetPropertyFactory.create(dataset=self.dataset, name='Central op', is_filter=False)
self.filter = DataSetPropertyFactory.create(name='CentralMvir rf', units="Msun/h", dataset=self.dataset)
self.computed_filter = DataSetPropertyFactory.create(name='Computed Property', dataset=self.dataset, is_computed=True)
self.sed = StellarModelFactory.create()
self.dust = DustModelFactory.create()
self.snapshot = SnapshotFactory.create(dataset=self.dataset, redshift='0.33')
self.band_pass_filters = [BandPassFilterFactory.create(), BandPassFilterFactory.create()]
self.output_paths = ['job1', 'large_job']
self.dir_paths = [os.path.join(settings.FILES_BASE, output_path) for output_path in self.output_paths]
txt_template = loader.get_template('jobs/light_cone_job-summary.txt')
summary_context = Context(self.make_parameters())
self.summary_text = txt_template.render(summary_context)
self.file_names_to_contents = {
'file1': 'abc\n',
'filez2.txt': 'pqr\n',
'file3': 'xyz\n',
'job2/fileA.html': 'aaaahhhh & aaaaa',
'job2/fileB': 'baaaaaa',
'summary.txt': self.summary_text,
}
self.file_names_to_contents2 = {
'waybigfile1': 'xnfaihnehrawlrwerajelrjxmjaeimrjwmrejlxaljrxm;kjmrlakjemrajlejrljrljaereirje;rjmriarie;rirjijeaim;jea;ljmxirjwra;ojer',
'waybigfile2': 'xnfaihnehrawlrwerajelrjxmjaeimrjwmrejlxaljrxm;kjmrlakjemrajlejrljrljaereirje;rjmriarie;rirjijeaim;jea;ljmxirjwra;ojer',
'waybigfile3': 'xnfaihnehrawlrwerajelrjxmjaeimrjwmrejlxaljrxm;kjmrlakjemrajlejrljrljaereirje;rjmriarie;rirjijeaim;jea;ljmxirjwra;ojer',
'waybigfile4': 'xnfaihnehrawlrwerajelrjxmjaeimrjwmrejlxaljrxm;kjmrlakjemrajlejrljrljaereirje;rjmriarie;rirjijeaim;jea;ljmxirjwra;ojer',
'waybigfile5': 'xnfaihnehrawlrwerajelrjxmjaeimrjwmrejlxaljrxm;kjmrlakjemrajlejrljrljaereirje;rjmriarie;rirjijeaim;jea;ljmxirjwra;ojer',
}
for file_name in self.file_names_to_contents.keys():
helper.create_file(self.dir_paths[0], file_name, self.file_names_to_contents)
for file_name in self.file_names_to_contents2.keys():
helper.create_file(self.dir_paths[1], file_name, self.file_names_to_contents2)
parameters = self.make_xml_parameters()
self.job.parameters = parameters
self.job.save()
self.completed_job = JobFactory.create(user=self.user, status=Job.COMPLETED, output_path=self.output_paths[0], parameters=parameters)
self.held_job = JobFactory.create(user=self.user, parameters=parameters, status=Job.HELD)
self.error_job = JobFactory.create(user=self.user, parameters=parameters, status=Job.ERROR, error_message="error")
self.in_progress_job = JobFactory.create(user=self.user, parameters=parameters, status=Job.IN_PROGRESS)
self.submitted_job = JobFactory.create(user=self.user, parameters=parameters, status=Job.SUBMITTED)
self.queued_job = JobFactory.create(user=self.user, parameters=parameters, status=Job.QUEUED)
def setUp(self):
super(DatasetTests, self).setUp()
s1 = SimulationFactory.create()
s2 = SimulationFactory.create()
s3 = SimulationFactory.create()
gm1 = GalaxyModelFactory.create()
gm2 = GalaxyModelFactory.create()
ds1 = DataSetFactory.create(simulation=s1, galaxy_model=gm1, max_job_box_count=25)
ds2 = DataSetFactory.create(simulation=s2, galaxy_model=gm2, max_job_box_count=25)
ds3 = DataSetFactory.create(simulation=s3, galaxy_model=gm2, max_job_box_count=25)
self.default_dataset = ds2
DataSetPropertyFactory.create(dataset=ds1, name="dataset property 1")
DataSetPropertyFactory.create(dataset=ds2, name="dataset property 2")
self.default_dataset.default_filter_field = DataSetPropertyFactory.create(
dataset=ds3, name="dataset property 3"
)
self.default_dataset.save()
SnapshotFactory.create(dataset=self.default_dataset, redshift="0.33")
self.survey_preset = SurveyPresetFactory.create(name="Preset 1", parameters="<xml></xml>")
for i in range(3):
StellarModelFactory.create(
label="stellar_label_%03d" % i, name="stellar_name_%03d" % i, description="<p>Description %d </p>" % i
)
BandPassFilterFactory.create(label="Band pass filter %03d" % i, filter_id="%d" % i)
DustModelFactory.create(
name="Dust_model_%03d.dat" % i, label="Dust model %03d" % i, details="<p>Detail %d </p>" % i
)
SnapshotFactory.create(dataset_id=i)
password = "******"
user = UserFactory.create(username="******", is_superuser=True)
user.set_password(password)
user.save()
self.login(user.username, password)
def setUp(self):
super(WorkflowTests, self).setUp()
OUTPUT_FORMATS = [
{'value':'csv', 'text':'CSV (Text2)', 'extension':'csv'},
{'value':'hdf5', 'text':'HDF5', 'extension':'hdf5'},
{'value': 'fits', 'text': 'FITS', 'extension': 'fits'},
{'value': 'votable', 'text': 'VOTable', 'extension': 'xml'}
]
self.output_formats = GlobalParameterFactory.create(parameter_name='output_formats', parameter_value=OUTPUT_FORMATS)
# "2012-12-13T13:55:36+10:00"
time.frozen_time = datetime.datetime(2012, 12, 20, 13, 55, 36, 0, UtcPlusTen())
self.user = UserFactory.create()
self.common_parameters = [
{'attrs': {'name': 'database-type'}, 'value': 'postgresql'},
{'attrs': {'name': 'database-host'}, 'value': 'tao02.hpc.swin.edu.au'},
{'attrs': {'name': 'database-name'}, 'value': 'millennium_full'},
{'attrs': {'name': 'database-port'}, 'value': '3306'},
{'attrs': {'name': 'database-user'}, 'value': ''},
{'attrs': {'name': 'database-pass'}, 'value': ''},
{
'attrs': {
'name': 'schema-version'
},
'value': '2.0',
},
]
self.simulation = SimulationFactory.create(box_size=500)
self.galaxy_model = GalaxyModelFactory.create()
self.dataset = DataSetFactory.create(simulation=self.simulation, galaxy_model=self.galaxy_model)
self.filter = DataSetPropertyFactory.create(name='CentralMvir rf', units="Msun/h", dataset=self.dataset)
self.computed_filter = DataSetPropertyFactory.create(name='Computed Filter', dataset=self.dataset, is_computed = True)
self.output_prop = DataSetPropertyFactory.create(name='Central op', dataset=self.dataset, is_filter=False)
self.snapshot = SnapshotFactory.create(dataset=self.dataset, redshift="0.1234567891")
self.stellar_model = StellarModelFactory.create(name='Stella')
self.band_pass_filter = BandPassFilterFactory.create(label='bandpass')
self.dust_model = DustModelFactory.create()
self.sed_parameters = {'apply_sed': True, 'single_stellar_population_model': self.stellar_model.id,
'band_pass_filters': [str(self.band_pass_filter.id) + '_apparent'], 'apply_dust': True,
'select_dust_model': self.dust_model.id}
self.sed_disabled = {'apply_sed': False}
self.sed_parameters_no_dust = {'apply_sed': True, 'single_stellar_population_model': self.stellar_model.id,
'band_pass_filters': [str(self.band_pass_filter.id) + '_absolute']}
# from code import interact
# interact(local=locals())
self.output_format = OUTPUT_FORMATS[0]['value']
self.output_format_parameters = {'supported_formats': self.output_format}
def setUp(self):
super(MockGalaxyFactoryTest, self).setUp()
GlobalParameterFactory.create(parameter_name='maximum-random-light-cones', parameter_value='10')
simulation = SimulationFactory.create(box_size=500)
simulation2 = SimulationFactory.create()
for unused in range(3):
g = GalaxyModelFactory.create()
ds = DataSetFactory.create(simulation=simulation, galaxy_model=g)
for unused in range(5):
DataSetPropertyFactory.create(dataset=ds)
SnapshotFactory.create(dataset=ds)
for unused in range(4):
g = GalaxyModelFactory.create()
ds = DataSetFactory.create(simulation=simulation2, galaxy_model=g)
dsp = DataSetPropertyFactory.create(dataset=ds)
ds.default_filter_field = dsp
ds.save()
for i in range(3):
StellarModelFactory.create()
BandPassFilterFactory.create(description='<p>BPF Description %s</p>' % i)
DustModelFactory.create()
self.survey_preset = SurveyPresetFactory.create(name='Preset 1', parameters='<xml></xml>')
username = "******"
password = "******"
self.user = UserFactory.create(username=username, password=password)
self.login(username, password)
self.visit('mock_galaxy_factory')
self.click('tao-tabs-' + 'light_cone')
self.select(self.lc_id('catalogue_geometry'), 'Light-Cone')
def setUp(self):
OUTPUT_FORMATS = [
{'value':'csv', 'text':'CSV (Text2)', 'extension':'csv'},
{'value':'hdf5', 'text':'HDF5', 'extension':'hdf5'},
{'value': 'fits', 'text': 'FITS', 'extension': 'fits'},
{'value': 'votable', 'text': 'VOTable', 'extension': 'xml'}
]
self.output_formats = GlobalParameterFactory.create(parameter_name='output_formats', parameter_value=OUTPUT_FORMATS)
self.simulation = SimulationFactory.create(box_size=500)
self.galaxy_model = GalaxyModelFactory.create()
self.dataset = DataSetFactory.create(simulation=self.simulation, galaxy_model=self.galaxy_model)
self.filter = DataSetPropertyFactory.create(name='CentralMvir rf', units="Msun/h", dataset=self.dataset)
self.computed_filter = DataSetPropertyFactory.create(name='Central Value', units="useless", dataset=self.dataset, is_computed=True)
self.output_prop = DataSetPropertyFactory.create(name='Central op', dataset=self.dataset, is_filter=False)
self.snapshot = SnapshotFactory.create(dataset=self.dataset, redshift='0.33')
self.stellar_model = StellarModelFactory.create()
self.band_pass_filter = BandPassFilterFactory.create()
self.dust_model = DustModelFactory.create()
def test_snapshot_choices(self):
from tao.datasets import snapshot_choices
s1 = SimulationFactory.create()
s2 = SimulationFactory.create()
g1 = GalaxyModelFactory.create()
g2 = GalaxyModelFactory.create()
g3 = GalaxyModelFactory.create()
d1 = DataSetFactory.create(simulation=s1, galaxy_model=g3)
d2 = DataSetFactory.create(simulation=s2, galaxy_model=g1)
d3 = DataSetFactory.create(simulation=s2, galaxy_model=g2)
snapshot1 = SnapshotFactory.create(dataset=d1, redshift='0.123')
snapshot2 = SnapshotFactory.create(dataset=d2, redshift='0.012')
snapshot3 = SnapshotFactory.create(dataset=d2, redshift='0.3')
snapshot4 = SnapshotFactory.create(dataset=d3, redshift='0.123')
snapshot5 = SnapshotFactory.create(dataset=d3, redshift='0.012')
snapshot6 = SnapshotFactory.create(dataset=d3, redshift='0.3')
self.assertEqual([
(snapshot1.id, snapshot1.redshift, {}),
],
snapshot_choices(d1.id))
self.assertEqual([
(snapshot2.id, snapshot2.redshift, {}),
(snapshot3.id, snapshot3.redshift, {}),
],
snapshot_choices(d2.id))
self.assertEqual([
(snapshot5.id, snapshot5.redshift, {}),
(snapshot4.id, snapshot4.redshift, {}),
(snapshot6.id, snapshot6.redshift, {})
],
snapshot_choices(d3.id))