def __init__(self, args=None, pool=None):
if args is None:
args = sys.argv[1:]
if isinstance(args, str):
args = args.split(' ')
parser = argparse.ArgumentParser(
description='Generate Assocation Data Products',
usage='asn_generate pool'
)
if pool is None:
parser.add_argument(
'pool', type=str, help='Association Pool'
)
op_group = parser.add_mutually_exclusive_group()
op_group.add_argument(
'-i', '--ids', nargs='+',
dest='asn_candidate_ids',
help='space-separated list of association candidate IDs to operate on.'
)
op_group.add_argument(
'--discover',
action='store_true',
help='Produce discovered associations'
)
op_group.add_argument(
'--all-candidates',
action='store_true', dest='all_candidates',
help='Produce all association candidate-specific associations'
)
parser.add_argument(
'-p', '--path', type=str,
default='.',
help='Folder to save the associations to. Default: "%(default)s"'
)
parser.add_argument(
'--save-orphans', dest='save_orphans',
nargs='?', const='orphaned.csv', default=False,
help='Save orphaned items into the specified table. Default: "%(default)s"'
)
parser.add_argument(
'--version-id', dest='version_id',
nargs='?', const=True, default=None,
help=(
'Version tag to add into association name and products.'
' If not specified, no version will be used.'
' If specified without a value, the current time is used.'
' Otherwise, the specified string will be used.'
)
)
parser.add_argument(
'-r', '--rules', action='append',
help='Association Rules file.'
)
parser.add_argument(
'--ignore-default', action='store_true',
help='Do not include default rules. -r should be used if set.'
)
parser.add_argument(
'--dry-run',
action='store_true', dest='dry_run',
help='Execute but do not save results.'
)
parser.add_argument(
'-d', '--delimiter', type=str,
default='|',
help='''Delimiter
to use if pool files are comma-separated-value
(csv) type files. Default: "%(default)s"
'''
)
parser.add_argument(
'--pool-format', type=str,
default='ascii',
help=(
'Format of the pool file.'
' Any format allowed by the astropy'
' Unified File I/O interface is allowed.'
' Default: "%(default)s"'
)
)
parser.add_argument(
'-v', '--verbose',
action='store_const', dest='loglevel',
const=logging.INFO, default=logging.NOTSET,
help='Output progress and results.'
)
parser.add_argument(
'-D', '--debug',
action='store_const', dest='loglevel',
const=logging.DEBUG,
help='Output detailed debugging information.'
)
parser.add_argument(
'--DMS',
action='store_true', dest='DMS_enabled',
help='Running under DMS workflow conditions.'
)
parser.add_argument(
'--format',
default='json',
help='Format of the association files. Default: "%(default)s"'
)
parser.add_argument(
'--version', action='version',
version='%(prog)s {}'.format(__version__),
help='Version of the generator.'
)
parser.add_argument(
'--no-merge', action='store_true',
help='Do not merge Level2 associations into one'
)
parsed = parser.parse_args(args=args)
# Configure logging
config = None
if parsed.DMS_enabled:
config = DMS_config
logger = log_config(name=__package__, config=config)
logger.setLevel(parsed.loglevel)
# Preamble
logger.info('Command-line arguments: {}'.format(args))
logger.context.set('asn_candidate_ids', parsed.asn_candidate_ids)
if pool is None:
logger.info('Reading pool {}'.format(parsed.pool))
self.pool = AssociationPool.read(
parsed.pool, delimiter=parsed.delimiter,
format=parsed.pool_format,
)
else:
self.pool = pool
# DMS: Add further info to logging.
try:
logger.context.set('program', self.pool[0]['PROGRAM'])
except KeyError:
pass
# Determine mode of operation. Options are
# 1) Only specified candidates
# 2) Only discovered assocations that do not match
# candidate associations
# 3) Both discovered and all candidate associations.
logger.info('Reading rules.')
if not parsed.discover and\
not parsed.all_candidates and\
parsed.asn_candidate_ids is None:
parsed.discover = True
parsed.all_candidates = True
if parsed.discover or parsed.all_candidates:
global_constraints = constrain_on_candidates(
None
)
elif parsed.asn_candidate_ids is not None:
global_constraints = constrain_on_candidates(
parsed.asn_candidate_ids
)
self.rules = AssociationRegistry(
parsed.rules,
include_default=not parsed.ignore_default,
global_constraints=global_constraints,
name=CANDIDATE_RULESET
)
if parsed.discover:
self.rules.update(
AssociationRegistry(
parsed.rules,
include_default=not parsed.ignore_default,
name=DISCOVER_RULESET
)
)
logger.info('Generating associations.')
self.associations = generate(
self.pool, self.rules, version_id=parsed.version_id
)
if parsed.discover:
logger.debug(
'# asns found before discover filtering={}'.format(
len(self.associations)
)
)
self.associations = filter_discovered_only(
self.associations,
DISCOVER_RULESET,
CANDIDATE_RULESET,
keep_candidates=parsed.all_candidates,
)
self.rules.Utility.resequence(self.associations)
# Do a grand merging. This is done particularly for
# Level2 associations.
if not parsed.no_merge:
try:
self.associations = self.rules.Utility.merge_asns(self.associations)
except AttributeError:
pass
logger.info(self.__str__())
if not parsed.dry_run:
self.save(
path=parsed.path,
format=parsed.format,
save_orphans=parsed.save_orphans
)
from jwst.associations import (
__version__,
AssociationPool,
AssociationRegistry,
generate,
)
from jwst.associations.lib.dms_base import DMSAttrConstraint
from jwst.associations.lib.constraint import (
ConstraintTrue,
)
from jwst.associations.lib.log_config import (log_config, DMS_config)
__all__ = ['Main']
# Configure logging
logger = log_config(name=__package__)
# Ruleset names
DISCOVER_RULESET = 'discover'
CANDIDATE_RULESET = 'candidate'
class Main():
"""
Generate Associations from an Association Pool
Parameters
----------
args: [str, ...], or None
The command line arguments. Can be one of
- `None`: `sys.argv` is then used.
import numpy as np
from jwst.associations import (
__version__,
AssociationPool,
AssociationRegistry,
generate,
)
from jwst.associations.lib.dms_base import DMSAttrConstraint
from jwst.associations.lib.constraint import (
ConstraintTrue, )
from jwst.associations.lib.log_config import (log_config, DMS_config)
# Configure logging
logger = log_config(name=__package__)
# Ruleset names
DISCOVER_RULESET = 'discover'
CANDIDATE_RULESET = 'candidate'
class Main():
"""
Generate Associations from an Association Pool
Parameters
----------
args: [str, ...], or None
The command line arguments. Can be one of
- `None`: `sys.argv` is then used.
def __init__(self, args=None, pool=None):
if args is None:
args = sys.argv[1:]
if isinstance(args, str):
args = args.split(' ')
parser = argparse.ArgumentParser(
description='Generate Assocation Data Products',
usage='asn_generate pool')
if pool is None:
parser.add_argument('pool', type=str, help='Association Pool')
op_group = parser.add_mutually_exclusive_group()
op_group.add_argument(
'-i',
'--ids',
nargs='+',
dest='asn_candidate_ids',
help=
'space-separated list of association candidate IDs to operate on.')
op_group.add_argument('--discover',
action='store_true',
help='Produce discovered associations')
op_group.add_argument(
'--all-candidates',
action='store_true',
dest='all_candidates',
help='Produce all association candidate-specific associations')
parser.add_argument(
'-p',
'--path',
type=str,
default='.',
help='Folder to save the associations to. Default: "%(default)s"')
parser.add_argument(
'--save-orphans',
dest='save_orphans',
nargs='?',
const='orphaned.csv',
default=False,
help=
'Save orphaned items into the specified table. Default: "%(default)s"'
)
parser.add_argument(
'--version-id',
dest='version_id',
nargs='?',
const=True,
default=None,
help=('Version tag to add into association name and products.'
' If not specified, no version will be used.'
' If specified without a value, the current time is used.'
' Otherwise, the specified string will be used.'))
parser.add_argument('-r',
'--rules',
action='append',
help='Association Rules file.')
parser.add_argument(
'--ignore-default',
action='store_true',
help='Do not include default rules. -r should be used if set.')
parser.add_argument('--dry-run',
action='store_true',
dest='dry_run',
help='Execute but do not save results.')
parser.add_argument('-d',
'--delimiter',
type=str,
default='|',
help='''Delimiter
to use if pool files are comma-separated-value
(csv) type files. Default: "%(default)s"
''')
parser.add_argument('--pool-format',
type=str,
default='ascii',
help=('Format of the pool file.'
' Any format allowed by the astropy'
' Unified File I/O interface is allowed.'
' Default: "%(default)s"'))
parser.add_argument('-v',
'--verbose',
action='store_const',
dest='loglevel',
const=logging.INFO,
default=logging.NOTSET,
help='Output progress and results.')
parser.add_argument('-D',
'--debug',
action='store_const',
dest='loglevel',
const=logging.DEBUG,
help='Output detailed debugging information.')
parser.add_argument('--DMS',
action='store_true',
dest='DMS_enabled',
help='Running under DMS workflow conditions.')
parser.add_argument(
'--format',
default='json',
help='Format of the association files. Default: "%(default)s"')
parser.add_argument('--version',
action='version',
version='%(prog)s {}'.format(__version__),
help='Version of the generator.')
parser.add_argument('--no-merge',
action='store_true',
help='Do not merge Level2 associations into one')
parsed = parser.parse_args(args=args)
# Configure logging
config = None
if parsed.DMS_enabled:
config = DMS_config
logger = log_config(name=__package__, config=config)
logger.setLevel(parsed.loglevel)
# Preamble
logger.info('Command-line arguments: {}'.format(args))
logger.context.set('asn_candidate_ids', parsed.asn_candidate_ids)
if pool is None:
logger.info('Reading pool {}'.format(parsed.pool))
self.pool = AssociationPool.read(
parsed.pool,
delimiter=parsed.delimiter,
format=parsed.pool_format,
)
else:
self.pool = pool
# DMS: Add further info to logging.
try:
logger.context.set('program', self.pool[0]['PROGRAM'])
except KeyError:
pass
# Determine mode of operation. Options are
# 1) Only specified candidates
# 2) Only discovered assocations that do not match
# candidate associations
# 3) Both discovered and all candidate associations.
logger.info('Reading rules.')
if not parsed.discover and\
not parsed.all_candidates and\
parsed.asn_candidate_ids is None:
parsed.discover = True
parsed.all_candidates = True
if parsed.discover or parsed.all_candidates:
global_constraints = constrain_on_candidates(None)
elif parsed.asn_candidate_ids is not None:
global_constraints = constrain_on_candidates(
parsed.asn_candidate_ids)
self.rules = AssociationRegistry(
parsed.rules,
include_default=not parsed.ignore_default,
global_constraints=global_constraints,
name=CANDIDATE_RULESET)
if parsed.discover:
self.rules.update(
AssociationRegistry(parsed.rules,
include_default=not parsed.ignore_default,
name=DISCOVER_RULESET))
logger.info('Generating associations.')
self.associations = generate(self.pool,
self.rules,
version_id=parsed.version_id)
if parsed.discover:
logger.debug('# asns found before discover filtering={}'.format(
len(self.associations)))
self.associations = filter_discovered_only(
self.associations,
DISCOVER_RULESET,
CANDIDATE_RULESET,
keep_candidates=parsed.all_candidates,
)
self.rules.Utility.resequence(self.associations)
# Do a grand merging. This is done particularly for
# Level2 associations.
if not parsed.no_merge:
try:
self.associations = self.rules.Utility.merge_asns(
self.associations)
except AttributeError:
pass
logger.info(self.__str__())
if not parsed.dry_run:
self.save(path=parsed.path,
format=parsed.format,
save_orphans=parsed.save_orphans)
def __init__(self, args=None):
if args is None:
args = sys.argv[1:]
if isinstance(args, str):
args = args.split(' ')
parser = argparse.ArgumentParser(
description='Generate Assocation Data Products',
usage='asn_generate pool'
)
parser.add_argument(
'pool', type=str, help='Association Pool'
)
parser.add_argument(
'-r', '--rules', action='append',
help='Association Rules file.'
)
parser.add_argument(
'-i', '--ids', nargs='+',
dest='asn_candidate_ids',
help='space-separated list of association candidate IDs to operate on.'
)
parser.add_argument(
'--ignore-default', action='store_true',
help='Do not include default rules. -r should be used if set.'
)
parser.add_argument(
'--dry_run',
action='store_true', dest='dry_run',
help='Execute but do not save results.'
)
parser.add_argument(
'-p', '--path', type=str,
default='.',
help='Folder to save the associations to. Default: "%(default)s"'
)
parser.add_argument(
'-d', '--delimiter', type=str,
default='|',
help='''Delimiter
to use if pool files are comma-separated-value
(csv) type files. Default: "%(default)s"
'''
)
parser.add_argument(
'-v', '--verbose',
action='store_const', dest='loglevel',
const=logging.INFO, default=logging.NOTSET,
help='Output progress and results.'
)
parser.add_argument(
'-D', '--debug',
action='store_const', dest='loglevel',
const=logging.DEBUG,
help='Output detailed debugging information.'
)
parser.add_argument(
'--DMS',
action='store_true', dest='DMS_enabled',
help='Running under DMS workflow conditions.'
)
parser.add_argument(
'--cross-candidate-only',
action='store_true', dest='cross_candidate_only',
help='Only produce cross-candidate associations'
)
parsed = parser.parse_args(args=args)
# Configure logging
config = None
if parsed.DMS_enabled:
config = DMS_config
logger = log_config(name=__package__, config=config)
logger.setLevel(parsed.loglevel)
# Preamble
logger.info('Command-line arguments: {}'.format(args))
logger.context.set('asn_candidate_ids', parsed.asn_candidate_ids)
logger.info('Reading pool {}'.format(parsed.pool))
self.pool = AssociationPool.read(
parsed.pool, delimiter=parsed.delimiter
)
# DMS: Add further info to logging.
try:
logger.context.set('program', self.pool[0]['PROGRAM'])
except KeyError:
pass
# Setup rules.
# Check for Candidate-specific or whole program.
# In DMS, this is the difference between Level 3
# versus Level 3.5 processing.
# The rules themselves do not contain this knowledge.
self.cross_candidate_mode = parsed.asn_candidate_ids is None
self.cross_candidate_only = parsed.cross_candidate_only
global_constraints = {}
if not self.cross_candidate_mode:
global_constraints['asn_candidate_ids'] = {
'value': parsed.asn_candidate_ids,
'inputs': ['ASN_CANDIDATE_ID', 'OBS_NUM'],
'force_unique': True,
}
logger.info('Reading rules.')
self.rules = AssociationRegistry(
parsed.rules,
include_default=not parsed.ignore_default,
global_constraints=global_constraints
)
logger.info('Generating associations.')
self.associations, self.orphaned = generate(self.pool, self.rules)
logger.debug(
'cross_candidate_mode="{}" cross_candidate_only="{}"'.format(
self.cross_candidate_mode,
self.cross_candidate_only
)
)
if self.cross_candidate_mode and self.cross_candidate_only:
self.associations = self.rules.Utility.filter_cross_candidates(
self.associations
)
logger.info(self.__str__())
if not parsed.dry_run:
self.save(path=parsed.path)
