def test_column_quantity():
# Regression test for pull request #356
# Previously Pipeline.__getitem__ would return column data from tables as
# an astropy.table.Column object. However, most functions take either
# numpy.ndarray or astropy.units.Quantity objects as arguments. As of
# astropy version 4.1.0 Column does not support all of the same class
# methods as Quantity e.g. to_value. This test ensures that column data in
# a Pipeline is accessed as either an ndarray or Quantity (depending on
# units). It also checks that functions using methods not supported by
# Column can be called on column data inside a Pipeline.
def value_in_cm(q):
return q.to_value(unit='cm')
config = {
'tables': {
'test_table': {
'lengths': Quantity(np.random.uniform(size=50), unit='m'),
'lengths_in_cm': Call(value_in_cm, [Ref('test_table.lengths')])}}}
pipeline = Pipeline(config)
pipeline.execute()
assert isinstance(pipeline['test_table.lengths'], Quantity)
assert isinstance(pipeline['test_table.lengths_in_cm'], np.ndarray)
np.testing.assert_array_less(0, pipeline['test_table.lengths_in_cm'])
np.testing.assert_array_less(pipeline['test_table.lengths_in_cm'], 100)
def test_multi_column_assignment_failure(na, nt):
# Test multi-column assignment failure with too few/many columns
config = {'tables': {
'multi_column_test_table': {
'a,b,c': Call(lambda nrows, ncols: np.ones((nrows, ncols)), [7, na]),
'd,e,f': Call(lambda nrows, ncols: (np.ones(nrows),) * ncols, [7, nt])}}}
pipeline = Pipeline(config)
with pytest.raises(ValueError):
pipeline.execute()
def test_multi_column_assignment():
# Test multi-column assignment from 2d arrays and tuples of 1d arrays
config = {'tables': {
'multi_column_test_table': {
'a,b ,c, d': Call(lambda nrows, ncols: np.ones((nrows, ncols)), [7, 4]),
'e , f, g': Call(lambda nrows, ncols: (np.ones(nrows),) * ncols, [7, 3]),
'h': Call(list, [Ref('multi_column_test_table.a')]),
'i': Call(list, [Ref('multi_column_test_table.f')])}}}
pipeline = Pipeline(config)
pipeline.execute()
def main(args=None):
parser = argparse.ArgumentParser(description="SkyPy pipeline driver")
parser.add_argument('--version', action='version', version=skypy_version)
parser.add_argument('config', help='Config file name')
parser.add_argument('-f',
'--format',
required=False,
choices=['fits', 'hdf5'],
help='Table file format')
parser.add_argument('-o',
'--overwrite',
action='store_true',
help='Whether to overwrite existing files')
# get system args if none passed
if args is None:
args = sys.argv[1:]
args = parser.parse_args(args or ['--help'])
config = load_skypy_yaml(args.config)
if 'lightcone' in config:
pipeline = Lightcone(config)
else:
pipeline = Pipeline(config)
pipeline.execute()
pipeline.write(file_format=args.format, overwrite=args.overwrite)
return (0)
def test_depends():
# Regression test for GitHub Issue #464
# Previously the .depends keyword was also being passed to functions as a
# keyword argument. This was because Pipeline was executing Item.infer to
# handle additional function arguments from context before handling
# additional dependencies specified using the .depends keyword. The
# .depends keyword is now handled first.
config = {'tables': {
'table_1': {
'column1': Call(np.random.uniform, [0, 1, 10])},
'table_2': {
'.init': Call(vstack, [], {
'tables': [Ref('table_1')],
'.depends': ['table_1.complete']})}}}
pipeline = Pipeline(config)
pipeline.execute()
assert np.all(pipeline['table_1'] == pipeline['table_2'])
def test_pipeline_read():
# Test reading config from a file
filename = get_pkg_data_filename('data/test_config.yml')
pipeline = Pipeline.read(filename)
pipeline.execute()
assert isinstance(pipeline['test_int'], int)
assert isinstance(pipeline['test_float'], float)
assert isinstance(pipeline['test_str'], str)
assert isinstance(pipeline['cosmology'], Cosmology)
assert isinstance(pipeline['test_table_1'], Table)
assert isinstance(pipeline['test_table_1']['test_column_3'], Column)
def test_pipeline_cosmology():
# Define function for testing pipeline cosmology
from skypy.utils import uses_default_cosmology
@uses_default_cosmology
def return_cosmology(cosmology):
return cosmology
# Initial default_cosmology
initial_default = default_cosmology.get()
# Test pipeline correctly sets default cosmology from parameters
# N.B. astropy cosmology class has not implemented __eq__ for comparison
H0, Om0 = 70, 0.3
config = {'parameters': {'H0': H0, 'Om0': Om0},
'cosmology': (FlatLambdaCDM, ['$H0', '$Om0']),
'test': (return_cosmology, ),
}
pipeline = Pipeline(config)
pipeline.execute()
assert type(pipeline['test']) == FlatLambdaCDM
assert pipeline['test'].H0.value == H0
assert pipeline['test'].Om0 == Om0
# Test pipeline correctly updates cosmology from new parameters
H0_new, Om0_new = 75, 0.25
pipeline.execute({'H0': H0_new, 'Om0': Om0_new})
assert type(pipeline['test']) == FlatLambdaCDM
assert pipeline['test'].H0.value == H0_new
assert pipeline['test'].Om0 == Om0_new
# Check that the astropy default cosmology is unchanged
assert default_cosmology.get() == initial_default
def test_call():
from skypy.pipeline import Pipeline
from skypy.pipeline._items import Call, Ref
# set up a mock pipeline
pipeline = Pipeline({})
# function we will call
def tester(arg1, arg2, *, kwarg1, kwarg2):
return arg1, arg2, kwarg1, kwarg2
# invalid construction
with pytest.raises(TypeError, match='function is not callable'):
Call(None, [], {})
with pytest.raises(TypeError, match='args is not a sequence'):
Call(tester, None, {})
with pytest.raises(TypeError, match='kwargs is not a mapping'):
Call(tester, [], None)
# good construction with no args or kwargs
call = Call(tester, [], {})
# call has incomplete args
with pytest.raises(TypeError, match=r'tester\(\)'):
call.evaluate(pipeline)
# good construction with arg1 and kwarg1
call = Call(tester, [1], {'kwarg1': 3})
# call still has incomplete args
with pytest.raises(TypeError, match=r'tester\(\)'):
call.evaluate(pipeline)
# infer required arg2 and kwarg2 from context
context = {
'arg2': 2,
'kwarg2': 4,
}
call.infer(context)
# call should be evaluatable now
result = call.evaluate(pipeline)
assert result == (1, 2, 3, 4)
# set up a call with references
call = Call(tester, [Ref('var1'), 2], {'kwarg1': Ref('var3'), 'kwarg2': 4})
# set up a pipeline with variables and a call that references them
pipeline = Pipeline({'var1': 1, 'var3': 3})
# check dependencies are resolved
deps = call.depend(pipeline)
assert deps == ['var1', 'var3']
# execute the pipeline (sets state) and evaluate the call
pipeline.execute()
result = call.evaluate(pipeline)
assert result == (1, 2, 3, 4)
def test_item():
from skypy.pipeline import Pipeline
from skypy.pipeline._items import Item
# set up a mock pipeline
pipeline = Pipeline({})
# construct the base class
item = Item()
# check `infer` method
item.infer({})
# check `depend` method
deps = item.depend(pipeline)
assert isinstance(deps, list)
assert len(deps) == 0
# check `evaluate` method
val = item.evaluate(pipeline)
assert val is None
def setup(options):
module_name = options['pipeline', 'current_module']
filename = options.get_string(option_section, 'pipeline')
pipeline = Pipeline.read(filename)
for parameter_name, parameter_value in pipeline.parameters.items():
register_new_parameter(options,
module_name,
parameter_name,
*parameter_expand(parameter_value))
results = options.get_string(option_section, 'results')
results_map = {}
for line in results.splitlines():
section_name, skypy_label = tuple(map(str.strip, line.split('=')))
section, _, name = section_name.partition('--')
results_map[section, name] = skypy_label
return module_name, pipeline, results_map
def test_hdf5():
size = 100
string = size*'a'
config = {'tables': {
'test_table': {
'column1': Call(np.random.uniform, [], {
'size': size}),
'column2': Call(np.random.uniform, [], {
'low': Ref('test_table.column1')}),
'column3': Call(list, [string], {})}}}
pipeline = Pipeline(config)
pipeline.execute()
pipeline.write('output.hdf5')
hdf_table = read_table_hdf5('output.hdf5', 'tables/test_table', character_as_bytes=False)
assert np.all(hdf_table == pipeline['test_table'])
def main(args=None):
parser = argparse.ArgumentParser(description="SkyPy pipeline driver")
parser.add_argument('--version', action='version', version=skypy_version)
parser.add_argument('config', help='Config file name')
parser.add_argument('output', help='Output file name')
parser.add_argument('-o', '--overwrite', action='store_true',
help='Whether to overwrite existing files')
parser.add_argument("-v", "--verbose", action="count", default=0,
help="Increase logging verbosity")
parser.add_argument("-q", "--quiet", action="count", default=0,
help="Decrease logging verbosity")
# get system args if none passed
if args is None:
args = sys.argv[1:]
args = parser.parse_args(args or ['--help'])
# Setup skypy logger
default_level = logging._nameToLevel['WARNING']
logging_level = default_level + 10 * (args.quiet - args.verbose)
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s')
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging_level)
stream_handler.setFormatter(formatter)
logger = logging.getLogger('skypy')
logger.setLevel(logging_level)
logger.addHandler(stream_handler)
try:
config = load_skypy_yaml(args.config)
pipeline = Pipeline(config)
pipeline.execute()
if args.output:
logger.info(f"Writing {args.output}")
pipeline.write(args.output, overwrite=args.overwrite)
except Exception as e:
logger.exception(e)
raise SystemExit(2) from e
return(0)
def test_pipeline_cosmology():
def return_cosmology(cosmology):
return cosmology
# Test pipeline correctly sets default cosmology from parameters
# N.B. astropy cosmology class has not implemented __eq__ for comparison
H0, Om0 = 70, 0.3
config = {'parameters': {'H0': H0, 'Om0': Om0},
'cosmology': Call(FlatLambdaCDM, [Ref('H0'), Ref('Om0')]),
'test': Call(return_cosmology),
}
pipeline = Pipeline(config)
pipeline.execute()
assert pipeline['test'] is pipeline['cosmology']
# Test pipeline correctly updates cosmology from new parameters
H0_new, Om0_new = 75, 0.25
pipeline.execute({'H0': H0_new, 'Om0': Om0_new})
assert pipeline['test'] is pipeline['cosmology']
def test_pipeline():
# Evaluate and store the default astropy cosmology.
config = {'test_cosmology': (default_cosmology.get, )}
pipeline = Pipeline(config)
pipeline.execute()
assert pipeline['test_cosmology'] == default_cosmology.get()
# Generate a simple two column table with a dependency. Also write the
# table to a fits file and check it's contents.
size = 100
string = size * 'a'
config = {
'tables': {
'test_table': {
'column1': (np.random.uniform, {
'size': size
}),
'column2': (np.random.uniform, {
'low': '$test_table.column1'
}),
'column3': (list, [string])
}
}
}
pipeline = Pipeline(config)
pipeline.execute()
pipeline.write(file_format='fits')
assert len(pipeline['test_table']) == size
assert np.all(
pipeline['test_table.column1'] < pipeline['test_table.column2'])
with fits.open('test_table.fits') as hdu:
assert np.all(Table(hdu[1].data) == pipeline['test_table'])
# Check for failure if output files already exist and overwrite is False
pipeline = Pipeline(config)
pipeline.execute()
with pytest.raises(OSError):
pipeline.write(file_format='fits', overwrite=False)
# Check that the existing output files are modified if overwrite is True
new_size = 2 * size
new_string = new_size * 'a'
config['tables']['test_table']['column1'][1]['size'] = new_size
config['tables']['test_table']['column3'][1][0] = new_string
pipeline = Pipeline(config)
pipeline.execute()
pipeline.write(file_format='fits', overwrite=True)
with fits.open('test_table.fits') as hdu:
assert len(hdu[1].data) == new_size
# Check for failure if 'column1' requires itself creating a cyclic
# dependency graph
config['tables']['test_table']['column1'] = (list, '$test_table.column1')
with pytest.raises(networkx.NetworkXUnfeasible):
Pipeline(config).execute()
# Check for failure if 'column1' and 'column2' both require each other
# creating a cyclic dependency graph
config['tables']['test_table']['column1'] = (list, '$test_table.column2')
with pytest.raises(networkx.NetworkXUnfeasible):
Pipeline(config).execute()
# Check for failure if 'column1' is removed from the config so that the
# requirements for 'column2' are not satisfied.
del config['tables']['test_table']['column1']
with pytest.raises(KeyError):
Pipeline(config).execute()
# Check variables intialised by value
config = {
'test_int': 1,
'test_float': 1.0,
'test_string': 'hello world',
'test_list': [0, 'one', 2.],
'test_dict': {
'a': 'b'
}
}
pipeline = Pipeline(config)
pipeline.execute()
assert isinstance(pipeline['test_int'], int)
assert isinstance(pipeline['test_float'], float)
assert isinstance(pipeline['test_string'], str)
assert isinstance(pipeline['test_list'], list)
assert isinstance(pipeline['test_dict'], dict)
assert pipeline['test_int'] == 1
assert pipeline['test_float'] == 1.0
assert pipeline['test_string'] == 'hello world'
assert pipeline['test_list'] == [0, 'one', 2.]
assert pipeline['test_dict'] == {'a': 'b'}
# Check variables intialised by function
config = {
'test_func': (list, 'hello world'),
'len_of_test_func': (len, '$test_func'),
'nested_references': (sum, [['$test_func', [' '], '$test_func'], []]),
'nested_functions': (list, (range, (len, '$test_func')))
}
pipeline = Pipeline(config)
pipeline.execute()
assert pipeline['test_func'] == list('hello world')
assert pipeline['len_of_test_func'] == len('hello world')
assert pipeline['nested_references'] == list('hello world hello world')
assert pipeline['nested_functions'] == list(range(len('hello world')))
# Check parameter initialisation
config = {'parameters': {'param1': 1.0}}
pipeline = Pipeline(config)
pipeline.execute()
assert pipeline['param1'] == 1.0
# Update parameter and re-run
new_parameters = {'param1': 5.0}
pipeline.execute(parameters=new_parameters)
assert pipeline['param1'] == new_parameters['param1']
from astropy.table import Table
from astropy.utils.data import download_file
from skypy.pipeline import Pipeline
# survey parameters
cosmology = FlatLambdaCDM(Om0=0.3, H0=70)
survey_area = u.Quantity('2.38 deg2')
# download catalogue
alhambra = download_file(
'http://svo2.cab.inta-csic.es/vocats/alhambra/download/alhambra.csv.gz',
cache=True)
# simulate with SkyPy
sim = Pipeline.read('fig_9.yml')
sim.execute()
# the model
def schechter_SF(M, z):
M_star = -21.00 - 1.03 * (z - 0.5)
phi_star = 10.**(-2.51 - 0.01 * (z - 0.5))
alpha = -1.29
M = np.reshape(M, (-1, 1))
return 0.4 * np.log(10) * phi_star * 10.**(
-0.4 * (M - M_star) * (alpha + 1)) * np.exp(-10.**(-0.4 *
(M - M_star)))
# four redshift bins
def execute(self, parameters={}):
# Lightcone parameters
z_min = self.lightcone_config['z_min']
z_max = self.lightcone_config['z_max']
n_slice = self.lightcone_config['n_slice']
params = {
'z_min': z_min,
'z_max': z_min,
'slice_z_min': None,
'slice_z_max': None,
'slice_z_mid': None,
}
# Additional user-defined parameters
params.update(parameters)
# Update config with ligthcone parameters and user parameters
if 'parameters' in self.config:
self.config['parameters'].update(params)
else:
self.config['parameters'] = params
# SkyPy Pipeline object
pipeline = Pipeline(self.config)
# Initialise empty tables
self.tables = {k: Table() for k in pipeline.table_config.keys()}
# Cosmology from pipeline
if pipeline.cosmology:
self.cosmology = pipeline.get_value(pipeline.cosmology)
else:
self.cosmology = default_cosmology.get()
# Calculate equispaced comoving distance slices in redshift space
chi_min = self.cosmology.comoving_distance(z_min)
chi_max = self.cosmology.comoving_distance(z_max)
chi = np.linspace(chi_min, chi_max, n_slice + 1)
chi_mid = (chi[:-1] + chi[1:]) / 2
z = [
z_at_value(self.cosmology.comoving_distance, c, z_min, z_max)
for c in chi[1:-1]
]
z_mid = [
z_at_value(self.cosmology.comoving_distance, c, z_min, z_max)
for c in chi_mid
]
redshift_slices = zip([z_min] + z, z + [z_max], z_mid)
# Simulate redshift slices and append results to tables
for slice_z_min, slice_z_max, slice_z_mid in redshift_slices:
slice_params = {
'slice_z_min': slice_z_min,
'slice_z_max': slice_z_max,
'slice_z_mid': slice_z_mid
}
pipeline.execute(parameters=slice_params)
for k, v in self.tables.items():
self.tables[k] = vstack((v, pipeline[k]))
def test_unknown_reference():
config = {'param1': Ref('param2')}
pipeline = Pipeline(config)
with pytest.raises(KeyError):
pipeline.execute()
config = {'mydict': {
'param1': Ref('mydict.param2')}}
pipeline = Pipeline(config)
with pytest.raises(KeyError):
pipeline.execute()
config = {'tables': {
'mytable': {
'mycolumn': [0, 1, 2]}},
'myvalue': Ref('mytable.myothercolumn')}
pipeline = Pipeline(config)
with pytest.raises(KeyError):
pipeline.execute()
# SDSS Photometry
# ---------------
#
# Here we compare the apparent magnitude distributions of our simulated
# galaxies with data from a :math:`10 \, \mathrm{deg^2}` region of the Sloan
# Digital Sky Survey [3]_. The binned SDSS magnitude distributions were
# generated from a query of the DR7 data release and can be downloaded
# :download:`here <../../../examples/galaxies/sdss_dered_10deg2.ecsv>`.
from astropy.table import Table, vstack
from matplotlib import pyplot as plt
import numpy as np
from skypy.pipeline import Pipeline
# Execute SkyPy galaxy photometry simulation pipeline
pipeline = Pipeline.read("sdss_photometry.yml")
pipeline.execute()
skypy_galaxies = vstack([pipeline['blue_galaxies'], pipeline['red_galaxies']])
# SDSS magnitude distributions for a 10 degree^2 region
sdss_data = Table.read("sdss_dered_10deg2.ecsv", format='ascii.ecsv')
# Plot magnitude distributions for SkyPy simulation and SDSS data
bins = np.linspace(14.95, 25.05, 102)
plt.hist(skypy_galaxies['mag_r'],
bins=bins,
alpha=0.5,
color='r',
label='SkyPy-r')
plt.hist(skypy_galaxies['mag_u'],
bins=bins,
