def set_state(self,
scanId,
config=None,
inmeta=None,
sdmfile=None,
sdmscan=None,
bdfdir=None,
prefsname=None):
""" Given metadata source, define state for a scanId.
"""
# TODO: define prefsname according to config and/or heuristics
prefs = preferences.Preferences(**preferences.parsepreffile(
self.preffile, name=prefsname, inprefs=self.inprefs))
st = state.State(inmeta=inmeta,
config=config,
inprefs=prefs,
lock=self.lock,
sdmfile=sdmfile,
sdmscan=sdmscan,
bdfdir=bdfdir)
logger.info('State set for scanId {0}. Requires {1:.1f} GB read and'
' {2:.1f} GPU-sec to search.'.format(
st.metadata.scanId, heuristics.total_memory_read(st),
heuristics.total_compute_time(st)))
self.states[scanId] = st
def __init__(self,
config=None,
sdmfile=None,
sdmscan=None,
bdfdir=None,
inprefs=None,
inmeta=None,
preffile=None,
name=None,
showsummary=True,
lock=None,
validate=True):
""" Initialize preference attributes with text file, preffile.
name can select preference set from within yaml file.
preferences are overloaded with inprefs.
Metadata source can be either:
1) Config object is a scan_config object (see evla_mcast library) or
2) sdmfile and sdmscan (optional bdfdir for reading from CBE).
inmeta is a dict with key-value pairs to overload metadata (e.g., to
mock metadata from a simulation)
validate argument will use assertions to test state.
"""
self.config = config
self.sdmscan = sdmscan
if sdmfile:
sdmfile = sdmfile.rstrip('/')
self.sdmfile = sdmfile
self.lock = lock
# set prefs according to inprefs type
if isinstance(inprefs, preferences.Preferences):
self.prefs = inprefs
else:
# default values will result in empty dict
prefs = preferences.parsepreffile(preffile,
inprefs=inprefs,
name=name)
self.prefs = preferences.Preferences(**prefs)
# TODO: not working
logger.parent.setLevel(getattr(logging, self.prefs.loglevel))
self.metadata = metadata.make_metadata(config=config,
sdmfile=sdmfile,
sdmscan=sdmscan,
inmeta=inmeta,
bdfdir=bdfdir)
if validate:
assert self.validate() is True
if showsummary:
self.summarize()
def __init__(self, config=None, sdmfile=None, sdmscan=None, bdfdir=None,
inprefs=None, inmeta=None, preffile=None, name=None,
showsummary=True, lock=None, validate=True):
""" Initialize preference attributes with text file, preffile.
name can select preference set from within yaml file.
preferences are overloaded with inprefs.
Metadata source can be either:
1) Config object is a scan_config object (see evla_mcast library) or
2) sdmfile and sdmscan (optional bdfdir for reading from CBE).
inmeta is a dict with key-value pairs to overload metadata (e.g., to
mock metadata from a simulation)
validate argument will use assertions to test state.
"""
from rfpipe import preferences, metadata
self.config = config
self.sdmscan = sdmscan
if sdmfile:
sdmfile = sdmfile.rstrip('/')
self.sdmfile = sdmfile
self.lock = lock
self._corrections = None
# set prefs according to inprefs type
if isinstance(inprefs, preferences.Preferences):
self.prefs = inprefs
else:
# default values will result in empty dict
prefs = preferences.parsepreffile(preffile, inprefs=inprefs,
name=name)
try:
self.prefs = preferences.Preferences(**prefs)
except TypeError as exc:
from fuzzywuzzy import fuzz
badarg = exc.args[0].split('\'')[1]
closeprefs = [pref for pref in list(preferences.Preferences().__dict__) if fuzz.ratio(badarg, pref) > 50]
raise TypeError("Preference {0} not recognized. Did you mean {1}?".format(badarg, ', '.join(closeprefs)))
# TODO: not working
logger.parent.setLevel(getattr(logging, self.prefs.loglevel))
self.metadata = metadata.make_metadata(config=config, sdmfile=sdmfile,
sdmscan=sdmscan, inmeta=inmeta,
bdfdir=bdfdir)
if validate:
assert self.validate() is True
if showsummary:
self.summarize()
def set_state(self,
scanId,
config=None,
inmeta=None,
sdmfile=None,
sdmscan=None,
bdfdir=None,
validate=True,
showsummary=True):
""" Given metadata source, define state for a scanId.
Uses metadata to set preferences used in preffile (prefsname).
Preferences are then overloaded with self.inprefs.
Will inject mock transient based on mockprob and other parameters.
"""
from rfpipe import preferences, state
prefsname = get_prefsname(inmeta=inmeta,
config=config,
sdmfile=sdmfile,
sdmscan=sdmscan,
bdfdir=bdfdir)
inprefs = preferences.parsepreffile(self.preffile,
name=prefsname,
inprefs=self.inprefs)
# alternatively, overload prefs with compiled rules (req Python>= 3.5)
# inprefs = {**inprefs, **heuristics.band_prefs(inmeta)}
st = state.State(inmeta=inmeta,
config=config,
inprefs=inprefs,
lock=self.lock,
sdmfile=sdmfile,
sdmscan=sdmscan,
bdfdir=bdfdir,
validate=validate,
showsummary=showsummary)
logger.info('State set for scanId {0}. Requires {1:.1f} GB read and'
' {2:.1f} GPU-sec to search.'.format(
st.metadata.scanId, heuristics.total_memory_read(st),
heuristics.total_compute_time(st)))
self.states[scanId] = st
def handle_config(self, config):
""" Triggered when obs comes in.
Downstream logic starts here.
"""
logger.info('Received complete configuration for {0}, '
'scan {1}, source {2}, intent {3}'.format(
config.scanId, config.scanNo, config.source,
config.scan_intent))
if self.pklfile:
with open(self.pklfile, 'ab') as pkl:
pickle.dump(config, pkl)
if self.preffile:
prefs = preferences.Preferences(
**preferences.parsepreffile(self.preffile, name='default'))
elastic.indexscan_config(config, preferences=prefs)
def __init__(self,
config=None,
sdmfile=None,
sdmscan=None,
bdfdir=None,
inprefs=None,
inmeta=None,
preffile=None,
name=None,
showsummary=True,
lock=None,
validate=True):
""" Initialize preference attributes with text file, preffile.
name can select preference set from within yaml file.
preferences are overloaded with inprefs.
Metadata source can be either:
1) Config object is a scan_config object (see evla_mcast library) or
2) sdmfile and sdmscan (optional bdfdir for reading from CBE).
inmeta is a dict with key-value pairs to overload metadata (e.g., to
mock metadata from a simulation)
validate argument will use assertions to test state.
"""
from rfpipe import preferences, metadata
self.config = config
self.sdmscan = sdmscan
if sdmfile:
sdmfile = sdmfile.rstrip('/')
self.sdmfile = sdmfile
self.lock = lock
self._corrections = None
# set prefs according to inprefs type
if isinstance(inprefs, preferences.Preferences):
self.prefs = inprefs
else:
# default values will result in empty dict
prefs = preferences.parsepreffile(preffile,
inprefs=inprefs,
name=name)
try:
self.prefs = preferences.Preferences(**prefs)
except TypeError as exc:
from fuzzywuzzy import fuzz
badarg = exc.args[0].split('\'')[1]
closeprefs = [
pref for pref in list(preferences.Preferences().__dict__)
if fuzz.ratio(badarg, pref) > 50
]
raise TypeError(
"Preference {0} not recognized. Did you mean {1}?".format(
badarg, ', '.join(closeprefs)))
# TODO: not working
logger.parent.setLevel(getattr(logging, self.prefs.loglevel))
self.metadata = metadata.make_metadata(config=config,
sdmfile=sdmfile,
sdmscan=sdmscan,
inmeta=inmeta,
bdfdir=bdfdir)
if validate:
assert self.validate() is True
if showsummary:
self.summarize()
def __init__(self,
config=None,
sdmfile=None,
sdmscan=None,
inprefs={},
inmeta={},
preffile=None,
name=None,
showsummary=True):
""" Initialize preference attributes with text file, preffile.
name can select preference set from within yaml file.
preferences are overloaded with inprefs.
Metadata source can be either:
1) Config object is a scan_config object (see evla_mcast library) or
2) sdmfile and sdmscan.
inmeta is a dict with key-value pairs to overload metadata (e.g., to
mock metadata from a simulation)
"""
self.config = config
self.sdmscan = sdmscan
if sdmfile:
sdmfile = sdmfile.rstrip('/')
self.sdmfile = sdmfile
if isinstance(inprefs, dict):
# get pipeline preferences as dict
prefs = preferences.parsepreffile(preffile)
# optionally overload preferences
for key in inprefs:
prefs[key] = inprefs[key]
for key in prefs:
logger.debug(key, prefs[key], type(prefs[key]))
self.prefs = preferences.Preferences(**prefs)
elif isinstance(inprefs, preferences.Preferences):
self.prefs = inprefs
else:
logger.warn('inprefs should be either a dictionary or \
preferences.Preferences object')
# TODO: is this working?
logger.parent.setLevel(getattr(logging, self.prefs.loglevel))
if isinstance(inmeta, dict):
# get metadata
if (self.sdmfile and self.sdmscan) and not self.config:
meta = metadata.sdm_metadata(sdmfile, sdmscan)
elif self.config and not (self.sdmfile or self.sdmscan):
# config datasource can be vys or simulated data
datasource = inmeta[
'datasource'] if 'datasource' in inmeta else 'vys'
meta = metadata.config_metadata(config, datasource=datasource)
else:
meta = {}
# optionally overload metadata
for key in inmeta:
meta[key] = inmeta[key]
for key in meta:
logger.debug(key, meta[key], type(meta[key]))
self.metadata = metadata.Metadata(**meta)
elif isinstance(inmeta, metadata.Metadata):
self.metadata = inmeta
else:
logger.warn('inmeta should be either a dictionary or '
'metadata.Metadata object')
if showsummary:
self.summarize()