Пример #1
0
    def set_state(self,
                  scanId,
                  config=None,
                  inmeta=None,
                  sdmfile=None,
                  sdmscan=None,
                  bdfdir=None,
                  prefsname=None):
        """ Given metadata source, define state for a scanId.
        """

        # TODO: define prefsname according to config and/or heuristics
        prefs = preferences.Preferences(**preferences.parsepreffile(
            self.preffile, name=prefsname, inprefs=self.inprefs))

        st = state.State(inmeta=inmeta,
                         config=config,
                         inprefs=prefs,
                         lock=self.lock,
                         sdmfile=sdmfile,
                         sdmscan=sdmscan,
                         bdfdir=bdfdir)

        logger.info('State set for scanId {0}. Requires {1:.1f} GB read and'
                    ' {2:.1f} GPU-sec to search.'.format(
                        st.metadata.scanId, heuristics.total_memory_read(st),
                        heuristics.total_compute_time(st)))

        self.states[scanId] = st
Пример #2
0
    def __init__(self,
                 config=None,
                 sdmfile=None,
                 sdmscan=None,
                 bdfdir=None,
                 inprefs=None,
                 inmeta=None,
                 preffile=None,
                 name=None,
                 showsummary=True,
                 lock=None,
                 validate=True):
        """ Initialize preference attributes with text file, preffile.
        name can select preference set from within yaml file.
        preferences are overloaded with inprefs.

        Metadata source can be either:
        1) Config object is a scan_config object (see evla_mcast library) or
        2) sdmfile and sdmscan (optional bdfdir for reading from CBE).

        inmeta is a dict with key-value pairs to overload metadata (e.g., to
        mock metadata from a simulation)
        validate argument will use assertions to test state.
        """

        self.config = config
        self.sdmscan = sdmscan
        if sdmfile:
            sdmfile = sdmfile.rstrip('/')
        self.sdmfile = sdmfile
        self.lock = lock

        # set prefs according to inprefs type
        if isinstance(inprefs, preferences.Preferences):
            self.prefs = inprefs
        else:
            # default values will result in empty dict
            prefs = preferences.parsepreffile(preffile,
                                              inprefs=inprefs,
                                              name=name)
            self.prefs = preferences.Preferences(**prefs)

        # TODO: not working
        logger.parent.setLevel(getattr(logging, self.prefs.loglevel))

        self.metadata = metadata.make_metadata(config=config,
                                               sdmfile=sdmfile,
                                               sdmscan=sdmscan,
                                               inmeta=inmeta,
                                               bdfdir=bdfdir)

        if validate:
            assert self.validate() is True

        if showsummary:
            self.summarize()
Пример #3
0
    def __init__(self, config=None, sdmfile=None, sdmscan=None, bdfdir=None,
                 inprefs=None, inmeta=None, preffile=None, name=None,
                 showsummary=True, lock=None, validate=True):
        """ Initialize preference attributes with text file, preffile.
        name can select preference set from within yaml file.
        preferences are overloaded with inprefs.

        Metadata source can be either:
        1) Config object is a scan_config object (see evla_mcast library) or
        2) sdmfile and sdmscan (optional bdfdir for reading from CBE).

        inmeta is a dict with key-value pairs to overload metadata (e.g., to
        mock metadata from a simulation)
        validate argument will use assertions to test state.
        """

        from rfpipe import preferences, metadata

        self.config = config
        self.sdmscan = sdmscan
        if sdmfile:
            sdmfile = sdmfile.rstrip('/')
        self.sdmfile = sdmfile
        self.lock = lock
        self._corrections = None

        # set prefs according to inprefs type
        if isinstance(inprefs, preferences.Preferences):
            self.prefs = inprefs
        else:
            # default values will result in empty dict
            prefs = preferences.parsepreffile(preffile, inprefs=inprefs,
                                              name=name)
            try:
                self.prefs = preferences.Preferences(**prefs)
            except TypeError as exc:
                from fuzzywuzzy import fuzz
                badarg = exc.args[0].split('\'')[1]
                closeprefs = [pref for pref in list(preferences.Preferences().__dict__) if fuzz.ratio(badarg, pref) > 50]
                raise TypeError("Preference {0} not recognized. Did you mean {1}?".format(badarg, ', '.join(closeprefs)))

        # TODO: not working
        logger.parent.setLevel(getattr(logging, self.prefs.loglevel))

        self.metadata = metadata.make_metadata(config=config, sdmfile=sdmfile,
                                               sdmscan=sdmscan, inmeta=inmeta,
                                               bdfdir=bdfdir)

        if validate:
            assert self.validate() is True

        if showsummary:
            self.summarize()
Пример #4
0
    def set_state(self,
                  scanId,
                  config=None,
                  inmeta=None,
                  sdmfile=None,
                  sdmscan=None,
                  bdfdir=None,
                  validate=True,
                  showsummary=True):
        """ Given metadata source, define state for a scanId.
        Uses metadata to set preferences used in preffile (prefsname).
        Preferences are then overloaded with self.inprefs.
        Will inject mock transient based on mockprob and other parameters.
        """

        from rfpipe import preferences, state

        prefsname = get_prefsname(inmeta=inmeta,
                                  config=config,
                                  sdmfile=sdmfile,
                                  sdmscan=sdmscan,
                                  bdfdir=bdfdir)

        inprefs = preferences.parsepreffile(self.preffile,
                                            name=prefsname,
                                            inprefs=self.inprefs)

        # alternatively, overload prefs with compiled rules (req Python>= 3.5)
        #        inprefs = {**inprefs, **heuristics.band_prefs(inmeta)}

        st = state.State(inmeta=inmeta,
                         config=config,
                         inprefs=inprefs,
                         lock=self.lock,
                         sdmfile=sdmfile,
                         sdmscan=sdmscan,
                         bdfdir=bdfdir,
                         validate=validate,
                         showsummary=showsummary)

        logger.info('State set for scanId {0}. Requires {1:.1f} GB read and'
                    ' {2:.1f} GPU-sec to search.'.format(
                        st.metadata.scanId, heuristics.total_memory_read(st),
                        heuristics.total_compute_time(st)))

        self.states[scanId] = st
Пример #5
0
    def handle_config(self, config):
        """ Triggered when obs comes in.
        Downstream logic starts here.
        """

        logger.info('Received complete configuration for {0}, '
                    'scan {1}, source {2}, intent {3}'.format(
                        config.scanId, config.scanNo, config.source,
                        config.scan_intent))

        if self.pklfile:
            with open(self.pklfile, 'ab') as pkl:
                pickle.dump(config, pkl)

        if self.preffile:
            prefs = preferences.Preferences(
                **preferences.parsepreffile(self.preffile, name='default'))
            elastic.indexscan_config(config, preferences=prefs)
Пример #6
0
    def __init__(self,
                 config=None,
                 sdmfile=None,
                 sdmscan=None,
                 bdfdir=None,
                 inprefs=None,
                 inmeta=None,
                 preffile=None,
                 name=None,
                 showsummary=True,
                 lock=None,
                 validate=True):
        """ Initialize preference attributes with text file, preffile.
        name can select preference set from within yaml file.
        preferences are overloaded with inprefs.

        Metadata source can be either:
        1) Config object is a scan_config object (see evla_mcast library) or
        2) sdmfile and sdmscan (optional bdfdir for reading from CBE).

        inmeta is a dict with key-value pairs to overload metadata (e.g., to
        mock metadata from a simulation)
        validate argument will use assertions to test state.
        """

        from rfpipe import preferences, metadata

        self.config = config
        self.sdmscan = sdmscan
        if sdmfile:
            sdmfile = sdmfile.rstrip('/')
        self.sdmfile = sdmfile
        self.lock = lock
        self._corrections = None

        # set prefs according to inprefs type
        if isinstance(inprefs, preferences.Preferences):
            self.prefs = inprefs
        else:
            # default values will result in empty dict
            prefs = preferences.parsepreffile(preffile,
                                              inprefs=inprefs,
                                              name=name)
            try:
                self.prefs = preferences.Preferences(**prefs)
            except TypeError as exc:
                from fuzzywuzzy import fuzz
                badarg = exc.args[0].split('\'')[1]
                closeprefs = [
                    pref for pref in list(preferences.Preferences().__dict__)
                    if fuzz.ratio(badarg, pref) > 50
                ]
                raise TypeError(
                    "Preference {0} not recognized. Did you mean {1}?".format(
                        badarg, ', '.join(closeprefs)))

        # TODO: not working
        logger.parent.setLevel(getattr(logging, self.prefs.loglevel))

        self.metadata = metadata.make_metadata(config=config,
                                               sdmfile=sdmfile,
                                               sdmscan=sdmscan,
                                               inmeta=inmeta,
                                               bdfdir=bdfdir)

        if validate:
            assert self.validate() is True

        if showsummary:
            self.summarize()
Пример #7
0
    def __init__(self,
                 config=None,
                 sdmfile=None,
                 sdmscan=None,
                 inprefs={},
                 inmeta={},
                 preffile=None,
                 name=None,
                 showsummary=True):
        """ Initialize preference attributes with text file, preffile.
        name can select preference set from within yaml file.
        preferences are overloaded with inprefs.

        Metadata source can be either:
        1) Config object is a scan_config object (see evla_mcast library) or
        2) sdmfile and sdmscan.

        inmeta is a dict with key-value pairs to overload metadata (e.g., to
        mock metadata from a simulation)
        """

        self.config = config
        self.sdmscan = sdmscan
        if sdmfile:
            sdmfile = sdmfile.rstrip('/')
        self.sdmfile = sdmfile

        if isinstance(inprefs, dict):
            # get pipeline preferences as dict
            prefs = preferences.parsepreffile(preffile)

            # optionally overload preferences
            for key in inprefs:
                prefs[key] = inprefs[key]

            for key in prefs:
                logger.debug(key, prefs[key], type(prefs[key]))

            self.prefs = preferences.Preferences(**prefs)
        elif isinstance(inprefs, preferences.Preferences):
            self.prefs = inprefs
        else:
            logger.warn('inprefs should be either a dictionary or \
                         preferences.Preferences object')

        # TODO: is this working?
        logger.parent.setLevel(getattr(logging, self.prefs.loglevel))

        if isinstance(inmeta, dict):
            # get metadata
            if (self.sdmfile and self.sdmscan) and not self.config:
                meta = metadata.sdm_metadata(sdmfile, sdmscan)
            elif self.config and not (self.sdmfile or self.sdmscan):
                # config datasource can be vys or simulated data
                datasource = inmeta[
                    'datasource'] if 'datasource' in inmeta else 'vys'
                meta = metadata.config_metadata(config, datasource=datasource)
            else:
                meta = {}

            # optionally overload metadata
            for key in inmeta:
                meta[key] = inmeta[key]

            for key in meta:
                logger.debug(key, meta[key], type(meta[key]))

            self.metadata = metadata.Metadata(**meta)
        elif isinstance(inmeta, metadata.Metadata):
            self.metadata = inmeta
        else:
            logger.warn('inmeta should be either a dictionary or '
                        'metadata.Metadata object')

        if showsummary:
            self.summarize()