def setUp(self): self.telstate = katsdptelstate.TelescopeState() self.telstate.clear() self.n_servers = 4 self.executor = concurrent.futures.ThreadPoolExecutor(self.n_servers) self.server_chans = 1024 self.bchan = 1100 self.echan = 1300 self._seq = 0 self.parameters = [ { 'product_names': { 'G': 'product_G', 'K': 'product_K', 'KCROSS': 'product_KCROSS', 'B': 'product_B{}'.format(i) }, 'channel_freqs': np.arange(self.server_chans) # only length matters } for i in range(self.n_servers) ] self.solution_stores = { 'K': CalSolutionStoreLatest('K'), 'B': CalSolutionStoreLatest('B'), 'G': CalSolutionStore('G') }
def setup_telstate(cls, namespace: str) -> katsdptelstate.TelescopeState: telstate = katsdptelstate.TelescopeState().view(namespace) n_ants = 3 telstate['n_chans'] = 4096 telstate['n_chans_per_substream'] = 1024 telstate['n_bls'] = n_ants * (n_ants + 1) * 2 return telstate
def __init__(self, cbid_stream_rdb_file): self._ts = katsdptelstate.TelescopeState() self._ts.load_from_file(cbid_stream_rdb_file) metfilename = '{}.met'.format(self._ts['capture_block_id'] + '_' + self._ts['stream_name']) super(MeerKATFlagProductMetExtractor, self).__init__(metfilename) self.product_type = 'MeerKATFlagProduct'
def get_gains(self): val = yield self.get_telstate() telstate_address = "{}:{}".format(*eval(val)) last_calibration = yield self.get_last_calibration_timestamp() telstate = katsdptelstate.TelescopeState(telstate_address) corrections = get_phaseup_corrections(telstate, last_calibration, 1.0, False) raise Return(corrections)
def setup(self): self.tempdir = tempfile.TemporaryDirectory() self.store = NpyFileChunkStore(self.tempdir.name) self.shape = (12, 96, len(ANTENNAS) * (len(ANTENNAS) + 1) * 2) self.telstate = katsdptelstate.TelescopeState() self._populate_telstate(self.telstate) self._open_patcher = mock.patch('katdal.open', autospec=True, side_effect=self._katdal_open) self._open_patcher.start()
def from_url(cls, url, chunk_store='auto', **kwargs): """Construct TelstateDataSource from URL (RDB file / REDIS server). Parameters ---------- url : string URL serving as entry point to dataset (typically RDB file or REDIS) chunk_store : :class:`katdal.ChunkStore` object, optional Chunk store for visibility data (obtained automatically by default, or set to None for metadata-only dataset) kwargs : dict, optional Extra keyword arguments passed to telstate view and chunk store init """ url_parts = urlparse.urlparse(url, scheme='file') # Merge key-value pairs from URL query with keyword arguments # of function (the latter takes precedence) url_kwargs = dict(urlparse.parse_qsl(url_parts.query)) url_kwargs.update(kwargs) kwargs = url_kwargs # Extract Redis database number if provided db = int(kwargs.pop('db', '0')) if url_parts.scheme == 'file': # RDB dump file telstate = katsdptelstate.TelescopeState() try: telstate.load_from_file(url_parts.path) except OSError as err: raise DataSourceNotFound(str(err)) elif url_parts.scheme == 'redis': # Redis server try: telstate = katsdptelstate.TelescopeState(url_parts.netloc, db) except katsdptelstate.ConnectionError as e: raise DataSourceNotFound(str(e)) telstate = view_capture_stream(telstate, **kwargs) if chunk_store == 'auto': chunk_store = _infer_chunk_store(url_parts, telstate, **kwargs) return cls(telstate, chunk_store, source_name=url_parts.geturl())
def telstate_detection(filename): """File is an .rdb files. Any .rdb files passed are assumed to be MeerKAT telescope products, and should be either an 'sdp.vis' or 'sdp.flags' stream. Parameters ---------- filename: string : full path to the product Returns ------- MetExtractor: class : A metadata extractor class to extract metadata from the rdb file. """ ts = katsdptelstate.TelescopeState() ts.load_from_file(filename) stream_name = ts['stream_name'] v = ts.view(stream_name) if v['stream_type'] == 'sdp.vis': return MeerKATTelescopeProductMetExtractor elif v['stream_type'] == 'sdp.flags': return MeerKATFlagProductMetExtractor raise ProductTypeDetectionError('%s not a recognisable stream type')
def test_rdb_support(self): telstate = katsdptelstate.TelescopeState() view, cbid, sn, _, _ = make_fake_data_source(telstate, self.store, (5, 16, 40), PREFIX) telstate['capture_block_id'] = cbid telstate['stream_name'] = sn # Save telstate to temp RDB file since RDBWriter needs a filename and not a handle rdb_filename = f'{cbid}_{sn}.rdb' temp_filename = os.path.join(self.tempdir, rdb_filename) with RDBWriter(temp_filename) as rdbw: rdbw.save(telstate) # Read the file back in and upload it to S3 with open(temp_filename, mode='rb') as rdb_file: rdb_data = rdb_file.read() rdb_url = urllib.parse.urljoin(self.store_url, self.store.join(cbid, rdb_filename)) self.store.create_array(cbid) self.store.complete_request('PUT', rdb_url, data=rdb_data) # Check that data source can be constructed from URL (with auto chunk store) source_from_url = TelstateDataSource.from_url(rdb_url, **self.store_kwargs) source_direct = TelstateDataSource(view, cbid, sn, self.store) assert_telstate_data_source_equal(source_from_url, source_direct)
def test_telescope_state_fetcher_override(telstate_fetcher, mock_responses) -> None: telstate2 = katsdptelstate.TelescopeState() telstate2['another_key'] = 'rfi_mask_ranges.h5' model = telstate_fetcher.get('another_key', DummyModel, telstate=telstate2) assert len(model.ranges) == 2
def setup(self): self.tempdir = tempfile.mkdtemp() self.store = NpyFileChunkStore(self.tempdir) self.telstate = katsdptelstate.TelescopeState() self.cbid = 'cb'
def from_url(cls, url, chunk_store='auto', **kwargs): """Construct TelstateDataSource from URL (RDB file / REDIS server). Parameters ---------- url : string URL serving as entry point to dataset (typically RDB file or REDIS) chunk_store : :class:`katdal.ChunkStore` object, optional Chunk store for visibility data (obtained automatically by default, or set to None for metadata-only dataset) kwargs : dict, optional Extra keyword arguments passed to init, telstate view, chunk store init """ url_parts = urllib.parse.urlparse(url, scheme='file') # Merge key-value pairs from URL query with keyword arguments # of function (the latter takes precedence) url_kwargs = dict(urllib.parse.parse_qsl(url_parts.query)) url_kwargs.update(kwargs) kwargs = url_kwargs # Extract Redis database number if provided db = int(kwargs.pop('db', '0')) if url_parts.scheme == 'file': # RDB dump file telstate = katsdptelstate.TelescopeState() try: telstate.load_from_file(url_parts.path) except (OSError, katsdptelstate.RdbParseError) as e: raise DataSourceNotFound(str(e)) from e elif url_parts.scheme == 'redis': # Redis server try: telstate = katsdptelstate.TelescopeState(url_parts.netloc, db) except katsdptelstate.ConnectionError as e: raise DataSourceNotFound(str(e)) from e elif url_parts.scheme in {'http', 'https'}: # Treat URL prefix as an S3 object store (with auth info in kwargs) store_url = urllib.parse.urljoin(url, '..') # Strip off parameters, query strings and fragments to get basic URL rdb_url = urllib.parse.urlunparse( (url_parts.scheme, url_parts.netloc, url_parts.path, '', '', '')) telstate = katsdptelstate.TelescopeState() try: rdb_store = S3ChunkStore(store_url, **kwargs) with rdb_store.request('GET', rdb_url) as response: telstate.load_from_file(io.BytesIO(response.content)) except ChunkStoreError as e: raise DataSourceNotFound(str(e)) from e # If the RDB file is opened via archive URL, use that URL and # corresponding S3 credentials or token to access the chunk store if chunk_store == 'auto' and not kwargs.get('s3_endpoint_url'): chunk_store = rdb_store else: raise DataSourceNotFound( f"Unknown URL scheme '{url_parts.scheme}' - " 'telstate expects file, redis, or http(s)') telstate, capture_block_id, stream_name = view_l0_capture_stream( telstate, **kwargs) if chunk_store == 'auto': chunk_store = infer_chunk_store(url_parts, telstate, **kwargs) # Remove these from kwargs since they have already been extracted by view_l0_capture_stream kwargs.pop('capture_block_id', None) kwargs.pop('stream_name', None) return cls(telstate, capture_block_id, stream_name, chunk_store, source_name=url_parts.geturl(), **kwargs)
local_redis = subprocess.Popen(shlex.split(launch_cmd), stderr=subprocess.PIPE, stdout=subprocess.PIPE) time.sleep(3) if local_redis.poll(): logger.error( "Failed to launch local Redis instance, terminating. %s", local_redis.communicate()) sys.exit() logger.info("Local Redis instance launched successfully") redis_host = 'localhost' redis_endpoint = '{}:{}'.format(redis_host, args.redis_port) else: redis_host = args.redis redis_endpoint = '{}:{}'.format(redis_host, args.redis_port) ts = katsdptelstate.TelescopeState(redis_endpoint) logger.info("Connected to Redis on %s. DB has %d existing keys", redis_endpoint, len(ts.keys())) r_str = "" for attr in h5_file['TelescopeState'].attrs: r_str += redis_gen_proto("SET", attr, h5_file['TelescopeState'].attrs[attr]) if redis_endpoint: redis_bulk_str(r_str, redis_host, args.redis_port) if not args.obj_only: for d_count, dset in enumerate(h5_file['TelescopeState'].keys()): st = time.time() r_str = "" d_val = h5_file['TelescopeState'][dset].value
def main(): args = parse_args() try: h5_file = h5py.File(args.file) except Exception as e: logger.error("Failed to open specified HDF5 file. {}".format(e)) sys.exit() try: data = h5_file['Data/correlator_data'] except KeyError: logger.error("This does not appear to be a valid MeerKAT HDF5 file") sys.exit() if args.redis_only: logger.warning("Building Redis DB only - no data will be written...") if args.redis is None: logger.info("Launching local Redis instance") try: launch_cmd = "/usr/bin/redis-server --port {}".format( args.redis_port) local_redis = subprocess.Popen(shlex.split(launch_cmd), stderr=subprocess.PIPE, stdout=subprocess.PIPE) except OSError: launch_cmd = "/usr/local/bin/redis-server --port {}".format( args.redis_port) local_redis = subprocess.Popen(shlex.split(launch_cmd), stderr=subprocess.PIPE, stdout=subprocess.PIPE) time.sleep(3) if local_redis.poll(): logger.error( "Failed to launch local Redis instance, terminating. {}". format(local_redis.communicate())) sys.exit() logger.info("Local Redis instance launched successfully") redis_host = 'localhost' else: redis_host = args.redis ts = katsdptelstate.TelescopeState( endpoint='{}:{}'.format(redis_host, args.redis_port)) logger.info("Connected to Redis on {}:{}. DB has {} existing keys".format( redis_host, args.redis_port, len(ts.keys()))) im_count = 0 r_str = "" for attr in h5_file['TelescopeState'].attrs: r_str += gen_redis_proto("SET", attr, h5_file['TelescopeState'].attrs[attr]) im_count += 1 redis_bulk_str(r_str, redis_host, args.redis_port) d_count = 0 for dset in h5_file['TelescopeState'].keys(): i_count = 0 st = time.time() r_str = "" d_val = h5_file['TelescopeState'][dset].value # much quicker to read it first and then iterate for (timestamp, pval) in d_val: packed_ts = struct.pack('>d', float(timestamp)) r_str += gen_redis_proto("ZADD", dset, "0", packed_ts + pval) i_count += 1 bss = time.time() redis_bulk_str(r_str, redis_host, args.redis_port) logger.info( "Added {} items in {}s to key {}. Bulk insert time: {}".format( i_count, time.time() - st, dset, (time.time() - bss))) d_count += 1 logger.info("Added {} ranged keys to TelescopeState".format(d_count)) if args.redis_only: if args.redis is None: logger.warning( "Terminating locally launched redis instance (also saves telstate to local dump.rdb)" ) try: cli_cmd = "/usr/bin/redis-cli -p {} SHUTDOWN SAVE".format( args.redis_port) subprocess.call(shlex.split(cli_cmd)) except OSError: cli_cmd = "/usr/local/bin/redis-cli -p {} SHUTDOWN SAVE".format( args.redis_port) subprocess.call(shlex.split(cli_cmd)) local_redis.terminate() sys.exit(0) cluster = rados.Rados(conffile=args.ceph_conf) cluster.connect() available_pools = cluster.list_pools() if args.pool not in available_pools: logger.error( "Specified pool {} not available in this cluster ({})".format( args.pool, available_pools)) sys.exit() ioctx = cluster.open_ioctx(args.pool) pool_stats = ioctx.get_stats() logger.info( "Connected to pool {}. Currently holds {} objects totalling {} GB". format(args.pool, pool_stats['num_objects'], pool_stats['num_bytes'] / 2**30)) (freq_chunk, obj_size) = get_freq_chunk(data.shape, args.obj_size) if args.ts_limit > 0: ts_limit = args.ts_limit else: ts_limit = data.shape[0] obj_count_per_ts = data.shape[1] // freq_chunk obj_count = ts_limit * obj_count_per_ts ts.add("obj_basename", args.basename) ts.add("obj_chunk_size", freq_chunk) ts.add("obj_size", obj_size) ts.add("obj_count", obj_count) ts.add("obj_pool", args.pool) f = open(args.ceph_conf, "r") ts.add("obj_ceph_conf", f.readlines()) f.close() logger.info("Inserted obj schema metadata into telstate") logger.info( "Processing {} timestamps into {} objects of size {} with basename {}". format(ts_limit, obj_count, obj_size, args.basename)) ts_index = 0 for ts_slice in data: st = time.time() (bytes_written, objs_written) = write_ts(ioctx, ts_index, ts_slice, args.basename, freq_chunk) et = time.time() - st if objs_written == obj_count_per_ts and bytes_written == ( obj_size * obj_count_per_ts): logger.info( "Stored ts index {} in {}s ({} objects totalling {}MBps)". format(ts_index, et, obj_count_per_ts, obj_count_per_ts * obj_size / (1024 * 1024) / et)) else: logger.error( "Failed to full write ts index {}. Wrote {}/{} objects and {}/{} bytes" .format(ts_index, objs_written, obj_count_per_ts, bytes_written, obj_size * obj_count_per_ts)) ts_index += 1 if ts_index >= ts_limit: logger.info("Reached specified ts limit ({}).".format(ts_limit)) break logger.info("Staging complete...") if args.redis is None: raw_input( "You have started a local Redis server. Hit enter to kill this and cleanup." ) local_redis.terminate()
def setUp(self) -> None: self.tmpdir = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmpdir) self.port = 7148 self.n_channels = 1024 self.spectra_per_heap = 256 # No data actually travels through these multicast groups; # it gets mocked out to use local TCP sockets instead. self.endpoints = endpoint.endpoint_list_parser(self.port)( '239.102.2.0+7:{}'.format(self.port)) self.tcp_acceptors = [ _make_listen_socket() for endpoint in self.endpoints ] self.tcp_endpoints = [ endpoint.Endpoint(*sock.getsockname()) for sock in self.tcp_acceptors ] self.n_bengs = 16 self.ticks_between_spectra = 8192 self.adc_sample_rate = 1712000000.0 self.heaps_per_stats = 6 self.channels_per_heap = self.n_channels // self.n_bengs self.channels_per_endpoint = self.n_channels // len(self.endpoints) attrs = { 'i0_tied_array_channelised_voltage_0x_n_chans': self.n_channels, 'i0_tied_array_channelised_voltage_0x_n_chans_per_substream': self.channels_per_heap, 'i0_tied_array_channelised_voltage_0x_spectra_per_heap': self.spectra_per_heap, 'i0_tied_array_channelised_voltage_0x_src_streams': ['i0_antenna_channelised_voltage'], 'i0_tied_array_channelised_voltage_0x_bandwidth': self.adc_sample_rate / 2, 'i0_tied_array_channelised_voltage_0x_center_freq': 3 * self.adc_sample_rate / 2, 'i0_antenna_channelised_voltage_ticks_between_spectra': self.ticks_between_spectra, 'i0_antenna_channelised_voltage_instrument_dev_name': 'i0', 'i0_sync_time': 111111111.0, 'i0_scale_factor_timestamp': self.adc_sample_rate } telstate = katsdptelstate.TelescopeState() for key, value in attrs.items(): telstate[key] = value stats_int_time = (self.heaps_per_stats * self.ticks_between_spectra * self.spectra_per_heap / self.adc_sample_rate) self.args = bf_ingest_server.parse_args([ '--cbf-spead=' + endpoint.endpoints_to_str(self.endpoints), '--channels=128:768', '--file-base=' + self.tmpdir, '--stream-name=i0_tied_array_channelised_voltage_0x', '--interface=lo', '--stats=239.102.3.0:7149', '--stats-int-time={}'.format(stats_int_time), '--stats-interface=lo' ], argparse.Namespace(telstate=telstate)) self.loop = asyncio.get_event_loop() self.patch_add_endpoint() self.patch_create_session_config() self.patch_session_factory()
def telstate() -> katsdptelstate.TelescopeState: """Telescope state with some attributes populated.""" telstate = katsdptelstate.TelescopeState() telstate['sdp_model_base_url'] = test_utils.BASE_URL telstate['model_key'] = 'rfi_mask_ranges.h5' return telstate