def test_hdf5():
size = 100
string = size*'a'
config = {'tables': {
'test_table': {
'column1': Call(np.random.uniform, [], {
'size': size}),
'column2': Call(np.random.uniform, [], {
'low': Ref('test_table.column1')}),
'column3': Call(list, [string], {})}}}
pipeline = Pipeline(config)
pipeline.execute()
pipeline.write('output.hdf5')
hdf_table = read_table_hdf5('output.hdf5', 'tables/test_table', character_as_bytes=False)
assert np.all(hdf_table == pipeline['test_table'])
def main(args=None):
parser = argparse.ArgumentParser(description="SkyPy pipeline driver")
parser.add_argument('--version', action='version', version=skypy_version)
parser.add_argument('config', help='Config file name')
parser.add_argument('output', help='Output file name')
parser.add_argument('-o', '--overwrite', action='store_true',
help='Whether to overwrite existing files')
parser.add_argument("-v", "--verbose", action="count", default=0,
help="Increase logging verbosity")
parser.add_argument("-q", "--quiet", action="count", default=0,
help="Decrease logging verbosity")
# get system args if none passed
if args is None:
args = sys.argv[1:]
args = parser.parse_args(args or ['--help'])
# Setup skypy logger
default_level = logging._nameToLevel['WARNING']
logging_level = default_level + 10 * (args.quiet - args.verbose)
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s')
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging_level)
stream_handler.setFormatter(formatter)
logger = logging.getLogger('skypy')
logger.setLevel(logging_level)
logger.addHandler(stream_handler)
try:
config = load_skypy_yaml(args.config)
pipeline = Pipeline(config)
pipeline.execute()
if args.output:
logger.info(f"Writing {args.output}")
pipeline.write(args.output, overwrite=args.overwrite)
except Exception as e:
logger.exception(e)
raise SystemExit(2) from e
return(0)
def test_pipeline():
# Evaluate and store the default astropy cosmology.
config = {'test_cosmology': (default_cosmology.get, )}
pipeline = Pipeline(config)
pipeline.execute()
assert pipeline['test_cosmology'] == default_cosmology.get()
# Generate a simple two column table with a dependency. Also write the
# table to a fits file and check it's contents.
size = 100
string = size * 'a'
config = {
'tables': {
'test_table': {
'column1': (np.random.uniform, {
'size': size
}),
'column2': (np.random.uniform, {
'low': '$test_table.column1'
}),
'column3': (list, [string])
}
}
}
pipeline = Pipeline(config)
pipeline.execute()
pipeline.write(file_format='fits')
assert len(pipeline['test_table']) == size
assert np.all(
pipeline['test_table.column1'] < pipeline['test_table.column2'])
with fits.open('test_table.fits') as hdu:
assert np.all(Table(hdu[1].data) == pipeline['test_table'])
# Check for failure if output files already exist and overwrite is False
pipeline = Pipeline(config)
pipeline.execute()
with pytest.raises(OSError):
pipeline.write(file_format='fits', overwrite=False)
# Check that the existing output files are modified if overwrite is True
new_size = 2 * size
new_string = new_size * 'a'
config['tables']['test_table']['column1'][1]['size'] = new_size
config['tables']['test_table']['column3'][1][0] = new_string
pipeline = Pipeline(config)
pipeline.execute()
pipeline.write(file_format='fits', overwrite=True)
with fits.open('test_table.fits') as hdu:
assert len(hdu[1].data) == new_size
# Check for failure if 'column1' requires itself creating a cyclic
# dependency graph
config['tables']['test_table']['column1'] = (list, '$test_table.column1')
with pytest.raises(networkx.NetworkXUnfeasible):
Pipeline(config).execute()
# Check for failure if 'column1' and 'column2' both require each other
# creating a cyclic dependency graph
config['tables']['test_table']['column1'] = (list, '$test_table.column2')
with pytest.raises(networkx.NetworkXUnfeasible):
Pipeline(config).execute()
# Check for failure if 'column1' is removed from the config so that the
# requirements for 'column2' are not satisfied.
del config['tables']['test_table']['column1']
with pytest.raises(KeyError):
Pipeline(config).execute()
# Check variables intialised by value
config = {
'test_int': 1,
'test_float': 1.0,
'test_string': 'hello world',
'test_list': [0, 'one', 2.],
'test_dict': {
'a': 'b'
}
}
pipeline = Pipeline(config)
pipeline.execute()
assert isinstance(pipeline['test_int'], int)
assert isinstance(pipeline['test_float'], float)
assert isinstance(pipeline['test_string'], str)
assert isinstance(pipeline['test_list'], list)
assert isinstance(pipeline['test_dict'], dict)
assert pipeline['test_int'] == 1
assert pipeline['test_float'] == 1.0
assert pipeline['test_string'] == 'hello world'
assert pipeline['test_list'] == [0, 'one', 2.]
assert pipeline['test_dict'] == {'a': 'b'}
# Check variables intialised by function
config = {
'test_func': (list, 'hello world'),
'len_of_test_func': (len, '$test_func'),
'nested_references': (sum, [['$test_func', [' '], '$test_func'], []]),
'nested_functions': (list, (range, (len, '$test_func')))
}
pipeline = Pipeline(config)
pipeline.execute()
assert pipeline['test_func'] == list('hello world')
assert pipeline['len_of_test_func'] == len('hello world')
assert pipeline['nested_references'] == list('hello world hello world')
assert pipeline['nested_functions'] == list(range(len('hello world')))
# Check parameter initialisation
config = {'parameters': {'param1': 1.0}}
pipeline = Pipeline(config)
pipeline.execute()
assert pipeline['param1'] == 1.0
# Update parameter and re-run
new_parameters = {'param1': 5.0}
pipeline.execute(parameters=new_parameters)
assert pipeline['param1'] == new_parameters['param1']
