def full_pool_rules(request):
"""Setup to use the full example pool and registry"""
pool_fname = t_path('data/mega_pool.csv')
pool = AssociationPool.read(pool_fname)
rules = AssociationRegistry()
return (pool, rules, pool_fname)
def dms_registry():
"""Create the registry"""
dms_test_rules_path = t_path(path.join('data', 'dms_rules.py'))
dms_registry = AssociationRegistry(
[dms_test_rules_path], include_default=False
)
return dms_registry
def test_read_assoc_defs_fromdefault():
rules = AssociationRegistry()
assert len(rules) >= 3
rule_names = helpers.get_rule_names(rules)
assert 'DMS_Level3_Base' not in rules
valid_rules = ['Asn_Lv3Image', 'Asn_Lv3WFSCMB']
for rule in valid_rules:
assert rule in rule_names
def test_valid():
rules = AssociationRegistry()
asn_file = helpers.t_path(
'data/test_image_asn.json'
)
with open(asn_file, 'r') as asn_fp:
asn = load_asn(asn_fp)
valid_schema_list = rules.validate(asn)
assert isinstance(valid_schema_list, list)
def test_include_bases():
"""Test for included bases"""
dms_test_rules_path = t_path(path.join('data', 'dms_rules.py'))
dms_registry = AssociationRegistry([dms_test_rules_path],
include_default=False,
include_bases=True)
assert len(dms_registry) > 1
assert 'DMSBaseMixin' in dms_registry
def test_read_assoc_defs():
rules = AssociationRegistry([helpers.t_path('data/asn_rules_set1.py')],
include_default=False)
assert len(rules) >= 2
rule_names = helpers.get_rule_names(rules)
assert 'DMS_Level3_Base_Set1' not in rules
valid_rules = ['Asn_Dither_Set1', 'Asn_WFS_Set1']
for rule in valid_rules:
assert rule in rule_names
def test_simple():
"""Test generate on simple registry"""
registry = AssociationRegistry([t_path('data/rules_basic.py')],
include_default=False)
pool = AssociationPool()
pool['value'] = ['row1', 'row2']
asns = generate(pool, registry)
assert len(asns) == 1
assert len(asns[0]['members']) == 2
def test_global_constraints(constraints, pool, n_asns):
"""Test that global constraints get applied to all rules"""
rules = AssociationRegistry(global_constraints=constraints)
assert len(rules) >= 3
for constraint in constraints:
for rule in rules:
assert constraint in rules[rule].GLOBAL_CONSTRAINT
pool = helpers.combine_pools(pool)
asns = generate(pool, rules)
assert len(asns) == n_asns
def test_cmdline_change_rules(tmpdir):
"""Command line change the rule"""
rule = 'Association'
path = tmpdir.join('test_asn.json')
inlist = ['a', 'b', 'c']
args = [
'-o', path.strpath,
'-r', rule,
]
args = args + inlist
Main(args)
with open(path.strpath, 'r') as fp:
asn = load_asn(fp, registry=AssociationRegistry(include_bases=True))
assert inlist == asn['members']
def test_multi_rules():
rule_files = [
helpers.t_path('data/asn_rules_set1.py'),
helpers.t_path('data/asn_rules_set2.py')
]
rules = AssociationRegistry(rule_files, include_default=False)
assert len(rules) == 4
rule_names = helpers.get_rule_names(rules)
assert 'DMS_Level3_Base_Set1' not in rule_names
assert 'DMS_Level3_Base_Set2' not in rule_names
valid_rules = [
'Asn_Dither_Set1', 'Asn_Dither_Set2', 'Asn_WFS_Set1', 'Asn_WFS_Set2'
]
for rule in valid_rules:
assert rule in rule_names
def test_level2_from_cmdline(tmpdir):
"""Create a level2 assocaition from the command line"""
rule = 'DMSLevel2bBase'
path = tmpdir.join('test_asn.json')
inlist = ['a', 'b', 'c']
args = [
'-o', path.strpath,
'-r', rule,
]
args = args + inlist
Main(args)
with open(path.strpath, 'r') as fp:
asn = load_asn(fp, registry=AssociationRegistry(include_bases=True))
assert asn['asn_rule'] == 'DMSLevel2bBase'
assert asn['asn_type'] == 'None'
products = asn['products']
assert len(products) == len(inlist)
for product in products:
assert product['name'] in inlist
members = product['members']
assert len(members) == 1
member = members[0]
assert member['expname'] == product['name']
assert member['exptype'] == 'science'
def registry_level2_only(global_constraints=None):
"""Get registry with only Level2 rules"""
return AssociationRegistry(definition_files=[level2_rule_path()],
include_default=False,
global_constraints=global_constraints)
def test_nodefs():
with pytest.raises(AssociationError):
rules = AssociationRegistry(include_default=False)
def test_registry_backref():
rules = AssociationRegistry()
for name, rule in rules.items():
assert rule.registry == rules
def test_invalid():
rules = AssociationRegistry()
with pytest.raises(AssociationNotValidError):
rules.validate({})
def __init__(self, args=None, pool=None):
if args is None:
args = sys.argv[1:]
if isinstance(args, str):
args = args.split(' ')
parser = argparse.ArgumentParser(
description='Generate Assocation Data Products',
usage='asn_generate pool')
if pool is None:
parser.add_argument('pool', type=str, help='Association Pool')
op_group = parser.add_mutually_exclusive_group()
op_group.add_argument(
'-i',
'--ids',
nargs='+',
dest='asn_candidate_ids',
help=
'space-separated list of association candidate IDs to operate on.')
op_group.add_argument('--discover',
action='store_true',
help='Produce discovered associations')
op_group.add_argument(
'--all-candidates',
action='store_true',
dest='all_candidates',
help='Produce all association candidate-specific associations')
parser.add_argument(
'-p',
'--path',
type=str,
default='.',
help='Folder to save the associations to. Default: "%(default)s"')
parser.add_argument(
'--save-orphans',
dest='save_orphans',
nargs='?',
const='orphaned.csv',
default=False,
help=
'Save orphaned items into the specified table. Default: "%(default)s"'
)
parser.add_argument(
'--version-id',
dest='version_id',
nargs='?',
const=True,
default=None,
help=('Version tag to add into association name and products.'
' If not specified, no version will be used.'
' If specified without a value, the current time is used.'
' Otherwise, the specified string will be used.'))
parser.add_argument('-r',
'--rules',
action='append',
help='Association Rules file.')
parser.add_argument(
'--ignore-default',
action='store_true',
help='Do not include default rules. -r should be used if set.')
parser.add_argument('--dry-run',
action='store_true',
dest='dry_run',
help='Execute but do not save results.')
parser.add_argument('-d',
'--delimiter',
type=str,
default='|',
help='''Delimiter
to use if pool files are comma-separated-value
(csv) type files. Default: "%(default)s"
''')
parser.add_argument('--pool-format',
type=str,
default='ascii',
help=('Format of the pool file.'
' Any format allowed by the astropy'
' Unified File I/O interface is allowed.'
' Default: "%(default)s"'))
parser.add_argument('-v',
'--verbose',
action='store_const',
dest='loglevel',
const=logging.INFO,
default=logging.NOTSET,
help='Output progress and results.')
parser.add_argument('-D',
'--debug',
action='store_const',
dest='loglevel',
const=logging.DEBUG,
help='Output detailed debugging information.')
parser.add_argument('--DMS',
action='store_true',
dest='DMS_enabled',
help='Running under DMS workflow conditions.')
parser.add_argument(
'--format',
default='json',
help='Format of the association files. Default: "%(default)s"')
parser.add_argument('--version',
action='version',
version='%(prog)s {}'.format(__version__),
help='Version of the generator.')
parser.add_argument('--no-merge',
action='store_true',
help='Do not merge Level2 associations into one')
parsed = parser.parse_args(args=args)
# Configure logging
config = None
if parsed.DMS_enabled:
config = DMS_config
logger = log_config(name=__package__, config=config)
logger.setLevel(parsed.loglevel)
# Preamble
logger.info('Command-line arguments: {}'.format(args))
logger.context.set('asn_candidate_ids', parsed.asn_candidate_ids)
if pool is None:
logger.info('Reading pool {}'.format(parsed.pool))
self.pool = AssociationPool.read(
parsed.pool,
delimiter=parsed.delimiter,
format=parsed.pool_format,
)
else:
self.pool = pool
# DMS: Add further info to logging.
try:
logger.context.set('program', self.pool[0]['PROGRAM'])
except KeyError:
pass
# Determine mode of operation. Options are
# 1) Only specified candidates
# 2) Only discovered assocations that do not match
# candidate associations
# 3) Both discovered and all candidate associations.
logger.info('Reading rules.')
if not parsed.discover and\
not parsed.all_candidates and\
parsed.asn_candidate_ids is None:
parsed.discover = True
parsed.all_candidates = True
if parsed.discover or parsed.all_candidates:
global_constraints = constrain_on_candidates(None)
elif parsed.asn_candidate_ids is not None:
global_constraints = constrain_on_candidates(
parsed.asn_candidate_ids)
self.rules = AssociationRegistry(
parsed.rules,
include_default=not parsed.ignore_default,
global_constraints=global_constraints,
name=CANDIDATE_RULESET)
if parsed.discover:
self.rules.update(
AssociationRegistry(parsed.rules,
include_default=not parsed.ignore_default,
name=DISCOVER_RULESET))
logger.info('Generating associations.')
self.associations = generate(self.pool,
self.rules,
version_id=parsed.version_id)
if parsed.discover:
logger.debug('# asns found before discover filtering={}'.format(
len(self.associations)))
self.associations = filter_discovered_only(
self.associations,
DISCOVER_RULESET,
CANDIDATE_RULESET,
keep_candidates=parsed.all_candidates,
)
self.rules.Utility.resequence(self.associations)
# Do a grand merging. This is done particularly for
# Level2 associations.
if not parsed.no_merge:
try:
self.associations = self.rules.Utility.merge_asns(
self.associations)
except AttributeError:
pass
logger.info(self.__str__())
if not parsed.dry_run:
self.save(path=parsed.path,
format=parsed.format,
save_orphans=parsed.save_orphans)
def env():
rules = AssociationRegistry()
exposure_path = helpers.t_path('data/exposures')
exposures = glob(os.path.join(exposure_path, '*.fits'))
return rules, exposures