def test_pool(tmpdir): pool = AssociationPool.read(POOL_FILE) assert len(pool) == 636 tmp_pool = str(tmpdir.mkdir(__name__).join('tmp_pool.csv')) pool.write(tmp_pool) roundtrip = AssociationPool.read(tmp_pool) assert len(pool) == len(roundtrip) assert set(pool.colnames) == set(roundtrip.colnames)
def full_pool_rules(request): """Setup to use the full example pool and registry""" pool_fname = t_path('data/mega_pool.csv') pool = AssociationPool.read(pool_fname) rules = AssociationRegistry() return (pool, rules, pool_fname)
def full_pool_rules(request): """Setup to use the full example pool and registry""" pool_fname = t_path('data/mega_pool.csv') pool = AssociationPool.read(pool_fname) rules = AssociationRegistry() return (pool, rules, pool_fname)
def test_level35_names(pool_file): rules = registry_level3_only() pool = AssociationPool.read(pool_file) asns = generate(pool, rules) for asn in asns: product_name = asn['products'][0]['name'] if asn['asn_rule'] == 'Asn_IFU': m = re.match(LEVEL3_PRODUCT_NAME_NO_OPTELEM_REGEX, product_name) else: m = re.match(LEVEL3_PRODUCT_NAME_REGEX, product_name) assert m is not None
def test_level3_names(pool_file, global_constraints): rules = registry_level3_only( global_constraints=global_constraints ) pool = AssociationPool.read(pool_file) asns = generate(pool, rules) for asn in asns: product_name = asn['products'][0]['name'] if asn['asn_rule'] == 'Asn_Lv3MIRMRS': m = re.match(LEVEL3_PRODUCT_NAME_NO_OPTELEM_REGEX, product_name) else: m = re.match(LEVEL3_PRODUCT_NAME_REGEX, product_name) assert m is not None assert m.groupdict()['acid'] == 'o002'
def test_duplicate_names(): """ For Level 3 association, there should be no association with the same product name. Generation should produce log messages indicating when duplicate names have been found. """ pool = AssociationPool.read(t_path('data/jw00632_dups.csv')) constrain_all_candidates = constrain_on_candidates(None) rules = registry_level3_only(global_constraints=constrain_all_candidates) with pytest.warns(RuntimeWarning): asns = generate(pool, rules) # There should only be one association left. assert len(asns) == 1
def test_multiple_optelems(pool_file): rules = registry_level3_only() pool = AssociationPool.read(pool_file) asns = generate(pool, rules) for asn in asns: product_name = asn['products'][0]['name'] if asn['asn_rule'] != 'Asn_Lv3MIRMRS': m = re.match(LEVEL3_PRODUCT_NAME_REGEX, product_name) assert m is not None try: value = '-'.join(asn.constraints['opt_elem2'].found_values) except KeyError: value = None if value in EMPTY: assert '-' not in m.groupdict()['opt_elem'] else: assert '-' in m.groupdict()['opt_elem']
def combine_pools(pools, **pool_kwargs): """Combine pools into a single pool Parameters ---------- pools: str, astropy.table.Table, [str|Table, ...] The pools to combine. Either a singleton is passed or and iterable can be passed. The entries themselves can be either a file path or an astropy.table.Table-like object. pool_kwargs: dict Other keyword arguments to pass to AssociationPool.read Returns ------- AssociationPool|astropy.table.Table The combined pool """ if not is_iterable(pools): pools = [pools] just_pools = [] for pool in pools: if not isinstance(pool, Table): pool = AssociationPool.read(pool, **pool_kwargs) just_pools.append(pool) if len(just_pools) > 1: mega_pool = vstack(just_pools, metadata_conflicts='silent') else: mega_pool = just_pools[0].copy(copy_data=True) # Replace OBS_NUM and ASN_CANDIDATE_ID with actual numbers, if # necessary expnum = Counter(start=0) obsnum = Counter(start=0) acid = Counter(start=999) local_env = locals() global_env = globals() for row in mega_pool: mega_pool[row.index] = [ parse_value(v, global_env=global_env, local_env=local_env) for v in row ] return mega_pool
def test_niriss_wfss(): """Test association properties for NIRISS WFSS""" pool = AssociationPool.read( t_path(path.join('data', 'jw87800_20180412T163456_pool.csv')) ) cmd_args = [ '--dry-run', '--D' ] results = Main( cmd_args, pool=pool ) asns = results.associations # Need 4 associations: image2, spec2, image3, spec3 assert len(asns) == 12 asn_types = [ asn['asn_type'] for asn in asns ] assert REQUIRED_ASN_TYPES == set(asn_types) # Arrange associations by type asn_by_type = { asn['asn_type']: asn for asn in asns } # Ensure catalog and segmentation map names are correct in the spec2 associations l3name = asn_by_type['image3']['products'][0]['name'] source_cat = l3name + '_cat.ecsv' segmap = l3name + '_segm.fits' for product in asn_by_type['spec2']['products']: members_by_type = { member['exptype']: member for member in product['members'] } assert members_by_type['sourcecat']['expname'] == source_cat assert members_by_type['segmap']['expname'] == segmap
def test_duplicate_generate(): """Test for duplicate/overwrite association The pool has two exposures, one without a valid `asn_candidate`, and one with a valid observation `asn_candidate`. When set with the "all candidates" constraint, only one association should be made. The prompt for this test was that three associations were being created, two of which were the observation candidate, with the second being a duplicate of the first. The third was an extraneous discovered candidate. """ pool = AssociationPool.read(t_path('data/pool_duplicate.csv')) constrain_all_candidates = constrain_on_candidates(None) rules = registry_level3_only(global_constraints=constrain_all_candidates) asns = generate(pool, rules) assert len(asns) == 1 asn = asns[0] assert asn['asn_type'] == 'image3' assert asn['asn_id'] == 'o029'
def __init__(self, args=None, pool=None): if args is None: args = sys.argv[1:] if isinstance(args, str): args = args.split(' ') parser = argparse.ArgumentParser( description='Generate Assocation Data Products', usage='asn_generate pool' ) if pool is None: parser.add_argument( 'pool', type=str, help='Association Pool' ) op_group = parser.add_mutually_exclusive_group() op_group.add_argument( '-i', '--ids', nargs='+', dest='asn_candidate_ids', help='space-separated list of association candidate IDs to operate on.' ) op_group.add_argument( '--discover', action='store_true', help='Produce discovered associations' ) op_group.add_argument( '--all-candidates', action='store_true', dest='all_candidates', help='Produce all association candidate-specific associations' ) parser.add_argument( '-p', '--path', type=str, default='.', help='Folder to save the associations to. Default: "%(default)s"' ) parser.add_argument( '--save-orphans', dest='save_orphans', nargs='?', const='orphaned.csv', default=False, help='Save orphaned items into the specified table. Default: "%(default)s"' ) parser.add_argument( '--version-id', dest='version_id', nargs='?', const=True, default=None, help=( 'Version tag to add into association name and products.' ' If not specified, no version will be used.' ' If specified without a value, the current time is used.' ' Otherwise, the specified string will be used.' ) ) parser.add_argument( '-r', '--rules', action='append', help='Association Rules file.' ) parser.add_argument( '--ignore-default', action='store_true', help='Do not include default rules. -r should be used if set.' ) parser.add_argument( '--dry-run', action='store_true', dest='dry_run', help='Execute but do not save results.' ) parser.add_argument( '-d', '--delimiter', type=str, default='|', help='''Delimiter to use if pool files are comma-separated-value (csv) type files. Default: "%(default)s" ''' ) parser.add_argument( '--pool-format', type=str, default='ascii', help=( 'Format of the pool file.' ' Any format allowed by the astropy' ' Unified File I/O interface is allowed.' ' Default: "%(default)s"' ) ) parser.add_argument( '-v', '--verbose', action='store_const', dest='loglevel', const=logging.INFO, default=logging.NOTSET, help='Output progress and results.' ) parser.add_argument( '-D', '--debug', action='store_const', dest='loglevel', const=logging.DEBUG, help='Output detailed debugging information.' ) parser.add_argument( '--DMS', action='store_true', dest='DMS_enabled', help='Running under DMS workflow conditions.' ) parser.add_argument( '--format', default='json', help='Format of the association files. Default: "%(default)s"' ) parser.add_argument( '--version', action='version', version='%(prog)s {}'.format(__version__), help='Version of the generator.' ) parser.add_argument( '--no-merge', action='store_true', help='Do not merge Level2 associations into one' ) parsed = parser.parse_args(args=args) # Configure logging config = None if parsed.DMS_enabled: config = DMS_config logger = log_config(name=__package__, config=config) logger.setLevel(parsed.loglevel) # Preamble logger.info('Command-line arguments: {}'.format(args)) logger.context.set('asn_candidate_ids', parsed.asn_candidate_ids) if pool is None: logger.info('Reading pool {}'.format(parsed.pool)) self.pool = AssociationPool.read( parsed.pool, delimiter=parsed.delimiter, format=parsed.pool_format, ) else: self.pool = pool # DMS: Add further info to logging. try: logger.context.set('program', self.pool[0]['PROGRAM']) except KeyError: pass # Determine mode of operation. Options are # 1) Only specified candidates # 2) Only discovered assocations that do not match # candidate associations # 3) Both discovered and all candidate associations. logger.info('Reading rules.') if not parsed.discover and\ not parsed.all_candidates and\ parsed.asn_candidate_ids is None: parsed.discover = True parsed.all_candidates = True if parsed.discover or parsed.all_candidates: global_constraints = constrain_on_candidates( None ) elif parsed.asn_candidate_ids is not None: global_constraints = constrain_on_candidates( parsed.asn_candidate_ids ) self.rules = AssociationRegistry( parsed.rules, include_default=not parsed.ignore_default, global_constraints=global_constraints, name=CANDIDATE_RULESET ) if parsed.discover: self.rules.update( AssociationRegistry( parsed.rules, include_default=not parsed.ignore_default, name=DISCOVER_RULESET ) ) logger.info('Generating associations.') self.associations = generate( self.pool, self.rules, version_id=parsed.version_id ) if parsed.discover: logger.debug( '# asns found before discover filtering={}'.format( len(self.associations) ) ) self.associations = filter_discovered_only( self.associations, DISCOVER_RULESET, CANDIDATE_RULESET, keep_candidates=parsed.all_candidates, ) self.rules.Utility.resequence(self.associations) # Do a grand merging. This is done particularly for # Level2 associations. if not parsed.no_merge: try: self.associations = self.rules.Utility.merge_asns(self.associations) except AttributeError: pass logger.info(self.__str__()) if not parsed.dry_run: self.save( path=parsed.path, format=parsed.format, save_orphans=parsed.save_orphans )
def __init__(self, args=None, pool=None): if args is None: args = sys.argv[1:] if isinstance(args, str): args = args.split(' ') parser = argparse.ArgumentParser( description='Generate Assocation Data Products', usage='asn_generate pool') if pool is None: parser.add_argument('pool', type=str, help='Association Pool') op_group = parser.add_mutually_exclusive_group() op_group.add_argument( '-i', '--ids', nargs='+', dest='asn_candidate_ids', help= 'space-separated list of association candidate IDs to operate on.') op_group.add_argument('--discover', action='store_true', help='Produce discovered associations') op_group.add_argument( '--all-candidates', action='store_true', dest='all_candidates', help='Produce all association candidate-specific associations') parser.add_argument( '-p', '--path', type=str, default='.', help='Folder to save the associations to. Default: "%(default)s"') parser.add_argument( '--save-orphans', dest='save_orphans', nargs='?', const='orphaned.csv', default=False, help= 'Save orphaned items into the specified table. Default: "%(default)s"' ) parser.add_argument( '--version-id', dest='version_id', nargs='?', const=True, default=None, help=('Version tag to add into association name and products.' ' If not specified, no version will be used.' ' If specified without a value, the current time is used.' ' Otherwise, the specified string will be used.')) parser.add_argument('-r', '--rules', action='append', help='Association Rules file.') parser.add_argument( '--ignore-default', action='store_true', help='Do not include default rules. -r should be used if set.') parser.add_argument('--dry-run', action='store_true', dest='dry_run', help='Execute but do not save results.') parser.add_argument('-d', '--delimiter', type=str, default='|', help='''Delimiter to use if pool files are comma-separated-value (csv) type files. Default: "%(default)s" ''') parser.add_argument('--pool-format', type=str, default='ascii', help=('Format of the pool file.' ' Any format allowed by the astropy' ' Unified File I/O interface is allowed.' ' Default: "%(default)s"')) parser.add_argument('-v', '--verbose', action='store_const', dest='loglevel', const=logging.INFO, default=logging.NOTSET, help='Output progress and results.') parser.add_argument('-D', '--debug', action='store_const', dest='loglevel', const=logging.DEBUG, help='Output detailed debugging information.') parser.add_argument('--DMS', action='store_true', dest='DMS_enabled', help='Running under DMS workflow conditions.') parser.add_argument( '--format', default='json', help='Format of the association files. Default: "%(default)s"') parser.add_argument('--version', action='version', version='%(prog)s {}'.format(__version__), help='Version of the generator.') parser.add_argument('--no-merge', action='store_true', help='Do not merge Level2 associations into one') parsed = parser.parse_args(args=args) # Configure logging config = None if parsed.DMS_enabled: config = DMS_config logger = log_config(name=__package__, config=config) logger.setLevel(parsed.loglevel) # Preamble logger.info('Command-line arguments: {}'.format(args)) logger.context.set('asn_candidate_ids', parsed.asn_candidate_ids) if pool is None: logger.info('Reading pool {}'.format(parsed.pool)) self.pool = AssociationPool.read( parsed.pool, delimiter=parsed.delimiter, format=parsed.pool_format, ) else: self.pool = pool # DMS: Add further info to logging. try: logger.context.set('program', self.pool[0]['PROGRAM']) except KeyError: pass # Determine mode of operation. Options are # 1) Only specified candidates # 2) Only discovered assocations that do not match # candidate associations # 3) Both discovered and all candidate associations. logger.info('Reading rules.') if not parsed.discover and\ not parsed.all_candidates and\ parsed.asn_candidate_ids is None: parsed.discover = True parsed.all_candidates = True if parsed.discover or parsed.all_candidates: global_constraints = constrain_on_candidates(None) elif parsed.asn_candidate_ids is not None: global_constraints = constrain_on_candidates( parsed.asn_candidate_ids) self.rules = AssociationRegistry( parsed.rules, include_default=not parsed.ignore_default, global_constraints=global_constraints, name=CANDIDATE_RULESET) if parsed.discover: self.rules.update( AssociationRegistry(parsed.rules, include_default=not parsed.ignore_default, name=DISCOVER_RULESET)) logger.info('Generating associations.') self.associations = generate(self.pool, self.rules, version_id=parsed.version_id) if parsed.discover: logger.debug('# asns found before discover filtering={}'.format( len(self.associations))) self.associations = filter_discovered_only( self.associations, DISCOVER_RULESET, CANDIDATE_RULESET, keep_candidates=parsed.all_candidates, ) self.rules.Utility.resequence(self.associations) # Do a grand merging. This is done particularly for # Level2 associations. if not parsed.no_merge: try: self.associations = self.rules.Utility.merge_asns( self.associations) except AttributeError: pass logger.info(self.__str__()) if not parsed.dry_run: self.save(path=parsed.path, format=parsed.format, save_orphans=parsed.save_orphans)