def __init__(self, name: str = None, mode: str = 'rb', source: FileList = None, vfs: VirtualFileSystem = None, *args, **kwargs): """Open/create a new TAR file. Args: name: Name of file. mode: Open mode. source: An object that provides a list of filenames to write into the TAR file. """ # store vfs self.vfs = vfs # mode? if mode != 'rb': raise ValueError('Modes other than rb are not supported.') # get list of files to zip source = get_object(source) self._files = source(name) # init stream self._out_stream = io.BytesIO() self._buffer = b'' # write to the BytesIO with no compression self._tarfile = tarfile.TarFile.open(fileobj=self._out_stream, mode='w')
def open_file(self, filename: str, mode: str) -> VFSFile: """Open a file. The handling class is chosen depending on the rootse in the filename. Args: filename (str): Name of file to open. mode (str): Opening mode. Returns: (IOBase) File like object for given file. """ # split root root, filename = VirtualFileSystem.split_root(filename) # does root exist? if root not in self._roots: raise ValueError("Could not find root {0} for file.".format(root)) # create file object from pyobs.object import get_object fd = get_object(self._roots[root], object_class=VFSFile, name=filename, mode=mode) # return it return fd
def __init__( self, roof: typing.Union[str, IRoof], telescope: typing.Union[str, ITelescope], flatfield: typing.Union[str, IFlatField], functions: dict, priorities: typing.Union[dict, SkyflatPriorities], min_exptime: float = 0.5, max_exptime: float = 5, timespan: float = 7200, filter_change: float = 30, count: int = 20, readout: dict = None, *args, **kwargs, ): """Init a new SkyFlats script. Args: roof: Roof to use telescope: Telescope to use flatfield: FlatFielder to use functions: Dict with solalt-exptime functions for all filters/binning priorities: SkyflatPriorities object that returns priorities min_exptime: Minimum exposure time for flats max_exptime: Maximum exposure time for flats timespan: Timespan from now that should be scheduled [s] filter_change: Time required for filter change [s] count: Number of flats to schedule readout: Dictionary with readout times (in sec) per binning (as BxB). """ Script.__init__(self, *args, **kwargs) # store modules self._roof = roof self._telescope = telescope self._flatfield = flatfield # stuff self._count = count # get archive and priorities priorities = get_object(priorities, SkyflatPriorities) # create scheduler self._scheduler = Scheduler( functions, priorities, self.observer, min_exptime=min_exptime, max_exptime=max_exptime, timespan=timespan, filter_change=filter_change, count=count, readout=readout, )
def __init__( self, archive: Union[Dict[str, Any], Archive], pipeline: Union[Dict[str, Any], Pipeline], worker_procs: int = 4, filenames_calib: str = FILENAME, min_flats: int = 10, store_local: Optional[str] = None, create_calibs: bool = True, calib_science: bool = True, **kwargs: Any, ): """Creates a Night object for reducing a given night. Args: archive: Archive to fetch images from and write results to. pipeline: Science pipeline. worker_procs: Number of worker processes. filenames_calib: Filename pattern for master calibration files. min_flats: Minimum number of raw frames to create flat field. store_local: If True, files are stored in given local directory instead of uploaded to archive. create_calibs: If False, no calibration files are created for night. calib_science: If False, no science frames are calibrated. """ # get archive and science pipeline self._archive = get_object(archive, Archive) self._pipeline = get_object(pipeline, Pipeline, archive=archive) # stuff self._worker_processes = worker_procs self._min_flats = min_flats self._store_local = store_local self._create_calibs = create_calibs self._calib_science = calib_science # cache for master calibration frames self._master_frames: Dict[Tuple[ImageType, str, str, Optional[str]], Image] = {} # default filename patterns self._fmt_calib = FilenameFormatter(filenames_calib)
def __init__( self, focuser: Union[str, IFocuser], camera: Union[str, IImageGrabber], series: FocusSeries, offset: bool = False, filters: Optional[Union[str, IFilters]] = None, filter_name: Optional[str] = None, binning: Optional[int] = None, **kwargs: Any, ): """Initialize a new auto focus system. Args: focuser: Name of IFocuser. camera: Name of ICamera. filters: Name of IFilters, if any. filter_name: Name of filter to set. offset: If True, offsets are used instead of absolute focus values. """ Module.__init__(self, **kwargs) # store focuser and camera self._focuser = focuser self._camera = camera self._filters = filters self._offset = offset self._abort = threading.Event() # create focus series self._series: FocusSeries = get_object(series, FocusSeries) # init camera settings mixin CameraSettingsMixin.__init__(self, filters=filters, filter_name=filter_name, binning=binning, **kwargs) # register exceptions if isinstance(camera, str): exc.register_exception( exc.RemoteError, 3, timespan=600, module=camera, callback=self._default_remote_error_callback) if isinstance(focuser, str): exc.register_exception( exc.RemoteError, 3, timespan=600, module=focuser, callback=self._default_remote_error_callback)
async def run(self, **kwargs: Any) -> None: """Move telescope to pointing.""" # get telescope log.info("Getting proxy for telescope...") telescope = await self.proxy(self._telescope, ITelescope) # pointing pointing = get_object(self._pointing, SkyFlatsBasePointing, observer=self.observer) # point await pointing(telescope) log.info("Finished pointing telescope.")
def __init__(self, site: str, night: str, archive: Union[dict, Archive], science: Union[dict, Pipeline], worker_procs: int = 4, filenames_calib: str = '{SITEID}{TELID}-{INSTRUME}-{DAY-OBS|date:}-' '{IMAGETYP}-{XBINNING}x{YBINNING}{FILTER|filter}.fits', flats_combine: Union[str, Image.CombineMethod] = Image.CombineMethod.MEDIAN, flats_min_raw: int = 10, *args, **kwargs): """Creates a Night object for reducing a given night. Args: site: Telescope site to use. night: Night to reduce. archive: Archive to fetch images from and write results to. science: Science pipeline. worker_procs: Number of worker processes. filenames_calib: Filename pattern for master calibration files. flats_combine: Method to combine flats. flats_min_raw: Minimum number of raw frames to create flat field. *args: **kwargs: """ # get archive and science pipeline self._archive = get_object(archive, Archive) self._science_pipeline = get_object(science, Pipeline) # stuff self._site = site self._night = night self._worker_processes = worker_procs self._flats_combine = Image.CombineMethod(flats_combine) if isinstance(flats_combine, str) else flats_combine self._flats_min_raw = flats_min_raw # cache for master calibration frames self._master_frames: Dict[Tuple[Type, str, str, Optional[str]], Image] = {} # default filename patterns self._fmt_calib = FilenameFormatter(filenames_calib)
def __init__(self, photometry: Union[dict, Photometry] = None, astrometry: Union[dict, Astrometry] = None, masks: Dict[str, Union[Image, str]] = None, filenames: str = None, *args, **kwargs): """Pipeline for science images. Args: photometry: Photometry object. If None, no photometry is performed. astrometry: Astrometry object. If None, no astrometry is performed. masks: Dictionary with masks to use for each binning given as, e.g., 1x1. *args: **kwargs: """ # get photometry and astrometry self._photometry = None if photometry is None else get_object( photometry, Photometry) self._astrometry = None if astrometry is None else get_object( astrometry, Astrometry) # masks self._masks = {} if masks is not None: for binning, mask in masks.items(): if isinstance(mask, Image): self._masks[binning] = mask elif isinstance(mask, str): self._masks[binning] = Image.from_file(mask) else: raise ValueError('Unknown mask format.') # default filename patterns if filenames is None: filenames = '{SITEID}{TELID}-{INSTRUME}-{DAY-OBS|date:}-{FRAMENUM|string:04d}-{IMAGETYP|type}01.fits' self._formatter = FilenameFormatter(filenames)
def __init__(self, source_detection: SourceDetection, radius_column: str = "radius", **kwargs: Any): """Initialize a new projection focus series. Args: source_detection: Photometry to use for estimating PSF sizes """ # stuff self._source_detection: SourceDetection = get_object( source_detection, SourceDetection) self._radius_col = radius_column self._data: List[Dict[str, float]] = []
async def __call__(self, image: Image) -> Image: """Processes an image and stores new exposure time in exp_time attribute. Args: image: Image to process. Returns: Original image. """ # get object source_detection = get_object(self._source_detection, SourceDetection) # do photometry and get copy of catalog catalog = (await source_detection(image)).catalog if catalog is None: log.info("No catalog found in image.") return image # sort catalog by peak flux catalog.sort("peak") # saturation level if "DET-SATU" in image.header and "DET-GAIN" in image.header: saturation = image.header["DET-SATU"] / image.header["DET-GAIN"] else: saturation = 50000 # get max peak flux that we allow max_peak = saturation * self._saturated # filter out all stars that are saturated catalog = catalog[catalog["peak"] <= max_peak] # get brightest star, get its peak flux and store its coordinates star = catalog[0] peak = star["peak"] log.info("Found peak of %.2f at %.1fx%.1f.", star["peak"], star["x"], star["y"]) self.coordinates = (star["x"], star["y"]) # get exposure time of image exp_time = image.header["EXPTIME"] # calculate new exposure time and return it self.exp_time = exp_time / (peak - self._bias) * (max_peak - self._bias) return image
def __init__( self, archive: Union[Dict[str, Any], Archive], site: str, instrument: str, filter_names: List[str], binnings: List[int], *args, **kwargs, ): SkyflatPriorities.__init__(self) self._archive = get_object(archive, Archive) self._filter_names = filter_names self._binnings = binnings self._site = site self._instrument = instrument
def __init__(self, archive: Union[Dict[str, Any], Archive], max_cache_size: int = 20, **kwargs: Any): """Init a new image calibration pipeline step. Args: archive: Archive to fetch calibration frames from. """ ImageProcessor.__init__(self, **kwargs) # store self._max_cache_size = max_cache_size # get archive self._archive = get_object(archive, Archive)
def __init__(self, flatfield: typing.Union[str, IFlatField], functions: typing.Dict[str, str], priorities: typing.Union[dict, SkyflatPriorities], min_exptime: float = 0.5, max_exptime: float = 5, timespan: float = 7200, filter_change: float = 30, count: int = 20, *args, **kwargs): """Initialize a new flat field scheduler. Args: flatfield: Flat field module to use functions: Dict with flat functions priorities: Class handling priorities min_exptime: Minimum exposure time [s] max_exptime: Maximum exposure time [s] timespan: Time to scheduler after start [s] filter_change: Time required for filter change [s] count: Number of flats to take per filter/binning """ Module.__init__(self, *args, **kwargs) # store self._flatfield = flatfield self._count = count # abort self._abort = threading.Event() # priorities prio = get_object(priorities, SkyflatPriorities) # create scheduler self._scheduler = Scheduler(functions, prio, self.observer, min_exptime=min_exptime, max_exptime=max_exptime, timespan=timespan, filter_change=filter_change, count=count)
def __init__(self, name: str = None, label: str = None, comm: Union[Comm, dict] = None, *args, **kwargs): """Initializes a new pyobs module. Args: name: Name of module. If None, ID from comm object is used. label: Label for module. If None, name is used. comm: Comm object to use """ Object.__init__(self, *args, **kwargs) # get list of client interfaces self._interfaces: List[Type] = [] self._methods: Dict[str, Tuple[Callable, inspect.Signature]] = {} self._get_interfaces_and_methods() # get configuration options, i.e. all parameters from c'tor self._config_options = self._get_config_options() # comm object self.comm: Comm if comm is None: self.comm = DummyComm() elif isinstance(comm, Comm): self.comm = comm elif isinstance(comm, dict): log.info('Creating comm object...') self.comm = get_object(comm) else: raise ValueError('Invalid Comm object') # name and label self._name: str = name if name is not None else self.comm.name self._label: str = label if label is not None else self._name
def open_file(self, filename: str, mode: str, compression: bool = None) -> VFSFile: """Open a file. The handling class is chosen depending on the rootse in the filename. Args: filename (str): Name of file to open. mode (str): Opening mode. compression (bool): Automatically (de)compress data if True. Automatically determine from filename if None. Returns: (IOBase) File like object for given file. """ from .gzippipe import GzipReader, GzipWriter # split root root, filename = VirtualFileSystem.split_root(filename) # does root exist? if root not in self._roots: raise ValueError('Could not find root {0} for file.'.format(root)) # create file object fd = get_object(self._roots[root], name=filename, mode=mode) # compression? if compression or (compression is None and os.path.splitext(filename)[1] in self._compression): # create pipe if 'w' in mode: fd = GzipWriter(fd, close_fd=True) else: fd = GzipReader(fd, close_fd=True) # return it return fd
def __init__(self, config: str, log_file: Optional[str] = None, log_level: str = "info", **kwargs: Any): """Initializes a pyobs application. Args: config: Name of config file. log_file: Name of log file, if any. log_level: Logging level. log_rotate: Whether to rotate the log files. """ # get config name without path and extension self._config = config # formatter for logging, and list of logging handlers formatter = logging.Formatter( "%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d %(message)s") handlers = [] # create stdout logging handler stream_handler = logging.StreamHandler() stream_handler.setFormatter(formatter) handlers.append(stream_handler) # create file logging handler, if log file is given if log_file is not None: # in Windows, append a FileHandler, otherwise we use a WatchedFileHandler, which works well with logrotate if platform.system() == "Windows": file_handler = logging.FileHandler(log_file) else: file_handler = logging.handlers.WatchedFileHandler(log_file) # add log file handler file_handler.setFormatter(formatter) handlers.append(file_handler) # basic setup logging.basicConfig(handlers=handlers, level=logging.getLevelName(log_level.upper())) logging.captureWarnings(True) warnings.simplefilter("always", DeprecationWarning) # disable tornado logger logging.getLogger("tornado.access").disabled = True # set pyobs logger global log log = logging.getLogger(__name__) # hack threading to set thread names on OS level self._hack_threading() # load config log.info("Loading configuration from {0:s}...".format(self._config)) with StringIO(pre_process_yaml(self._config)) as f: cfg: Dict[str, Any] = yaml.safe_load(f) # create module and open it log.info("Creating module...") self._module = get_object(cfg, Module)
def __init__(self, functions: Dict[str, Union[str, Dict[str, str]]] = None, target_count: float = 30000, min_exptime: float = 0.5, max_exptime: float = 5, test_frame: tuple = None, counts_frame: tuple = None, allowed_offset_frac: float = 0.2, min_counts: int = 100, pointing: Union[dict, SkyFlatsBasePointing] = None, combine_binnings: bool = True, observer: Observer = None, vfs: VirtualFileSystem = None, callback: Callable = None, *args, **kwargs): """Initialize a new flat fielder. Depending on the value of combine_binnings, functions must be in a specific format: 1. combine_binnings=True: functions must be a dictionary of filter->function pairs, like {'clear': 'exp(-0.9*(h+3.9))'} In this case it is assumed that the average flux per pixel is directly correlated to the binning, i.e. a flat with 3x3 binning hast on average 9 times as much flux per pixel. 2. combine_binnings=False: functions must be nested one level deeper within the binning, like {'1x1': {'clear': 'exp(-0.9*(h+3.9))'}} Args: functions: Function f(h) for each filter to describe ideal exposure time as a function of solar elevation h, i.e. something like exp(-0.9*(h+3.9)) target_count: Count rate to aim for. min_exptime: Minimum exposure time. max_exptime: Maximum exposure time. test_frame: Tupel (left, top, width, height) in percent that describe the frame for on-sky testing. counts_frame: Tupel (left, top, width, height) in percent that describe the frame for calculating mean count rate. allowed_offset_frac: Offset from target_count (given in fraction of it) that's still allowed for good flat-field min_counts: Minimum counts in frames. combine_binnings: Whether different binnings use the same functions. observer: Observer to use. vfs: VFS to use. callback: Callback function for statistics. """ # store stuff self._target_count = target_count self._min_exptime = min_exptime self._max_exptime = max_exptime self._test_frame = (45, 45, 10, 10) if test_frame is None else test_frame self._counts_frame = (25, 25, 75, 75) if counts_frame is None else counts_frame self._allowed_offset_frac = allowed_offset_frac self._min_counts = min_counts self._combine_binnings = combine_binnings self._observer = observer self._vfs = vfs self._callback = callback # parse function if functions is None: functions = {} self._functions: Dict[Union[str, Tuple[str, str]], Any] if combine_binnings: # in the simple case, the key is just the filter self._functions = { filter_name: Parser().parse(func) for filter_name, func in functions.items() } else: # in case of separate binnings, the key to the functions dict is a tuple of binning and filter self._functions = {} for binning, func in functions.items(): # func must be a dict if isinstance(func, dict): for filter_name, func in func.items(): self._functions[binning, filter_name] = Parser().parse(func) else: raise ValueError( 'functions must be a dict of binnings, of combine_binnings is False.' ) # abort event self._abort = threading.Event() # pointing self._pointing = get_object(pointing, SkyFlatsBasePointing, observer=self._observer) # state machine self._state = FlatFielder.State.INIT # current exposure time self._exptime = None # median of last image self._median = None # exposures to do self._exposures_total = 0 self._exposures_done = 0 self._exptime_done = 0 # bias level self._bias_level = None # which twilight are we in? self._twilight = None # current request self._cur_filter: Optional[str] = None self._cur_binning: Optional[int] = None