def parse_args() -> argparse.Namespace:
parser = katsdpservices.ArgumentParser()
parser.add_argument('--url', type=str, help='WebSocket URL to connect to')
parser.add_argument('--namespace',
type=str,
help='Namespace to create in katportal [UUID]')
parser.add_argument('--receptors',
type=str,
help='Comma-separated list of receptors')
parser.add_argument('-a',
'--host',
type=str,
metavar='HOST',
default='',
help='Hostname to bind for katcp interface')
parser.add_argument('-p',
'--port',
type=int,
metavar='N',
default=2047,
help='Port to bind for katcp interface [%(default)s]')
parser.add_aiomonitor_arguments()
args = parser.parse_args()
# Can't use required= on the parser, because telstate-provided arguments
# are treated only as defaults rather than having set a value.
if args.telstate is None:
parser.error('argument --telstate is required')
if args.url is None:
parser.error('argument --url is required')
# TODO: make --receptors required too, once katsdpcontroller is updated
return args
def parse_args():
parser = katsdpservices.ArgumentParser()
parser.add_argument('file',
type=str,
nargs=1,
metavar='FILE',
help='HDF5 file to process')
parser.add_argument('--base-name',
type=str,
metavar='BASENAME',
help='Base name for objects (should include bucket '
'name for S3 object store)')
parser.add_argument('--obj-size',
type=float,
default=2.0,
help='Target object size in MB')
parser.add_argument('--max-dumps',
type=int,
default=0,
help='Number of dumps to process. Default is all.')
parser.add_argument(
'--ceph-conf',
type=str,
default="/etc/ceph/ceph.conf",
metavar='CEPHCONF',
help='Ceph configuration file used for cluster connect')
parser.add_argument('--ceph-pool',
type=str,
metavar='POOL',
help='Ceph pool to use for object storage')
parser.add_argument('--ceph-keyring',
help='Ceph keyring to use for object storage')
parser.add_argument('--s3-url',
type=str,
help='S3 endpoint URL (includes leading "http")')
parser.add_argument('--redis',
type=str,
help='Redis host to connect to as Telescope State. '
'Default is to start a new local instance.')
parser.add_argument('--redis-port',
type=int,
default=6379,
help='Port to use when connecting to Redis instance '
'(or creating a new one)')
parser.add_argument('--redis-only',
action='store_true',
help='Only (re)build Redis DB - no object creation')
parser.add_argument('--obj-only',
action='store_true',
help='Only populate object store - no Redis update')
args = parser.parse_args()
if not args.redis_only:
use_s3 = args.s3_url is not None
use_rados = args.ceph_pool is not None
if use_rados and use_s3:
parser.error('Please specify either --ceph-pool or --s3-*')
if args.base_name is None:
args.base_name = args.file[0].split(".")[0]
return args
async def run(loop: asyncio.AbstractEventLoop, server: VisibilityWriterServer) -> None:
await server.start()
for sig in [signal.SIGINT, signal.SIGTERM]:
loop.add_signal_handler(sig, lambda: on_shutdown(loop, server))
logger.info("Started visibility writer server.")
await server.join()
if __name__ == '__main__':
katsdpservices.setup_logging()
logger = logging.getLogger('vis_writer')
katsdpservices.setup_restart()
parser = katsdpservices.ArgumentParser()
add_common_args(parser)
parser.add_argument('--l0-spead', default=':7200', metavar='ENDPOINTS',
type=katsdptelstate.endpoint.endpoint_list_parser(7200),
help='Source port/multicast groups for L0 SPEAD stream. '
'[default=%(default)s]')
parser.add_argument('--l0-interface', metavar='INTERFACE',
help='Network interface to subscribe to for L0 stream. '
'[default=auto]')
parser.add_argument('--l0-name', default='sdp_l0', metavar='NAME',
help='Name of L0 stream from ingest [default=%(default)s]')
parser.add_argument('--l0-ibv', action='store_true',
help='Use ibverbs acceleration to receive L0 stream [default=no]')
parser.set_defaults(telstate='localhost', port=2046)
args = parser.parse_args()
def parse_args():
parser = katsdpservices.ArgumentParser()
parser.add_argument('--file',
type=str,
default=None,
metavar='FILE',
help='h5 file to process.')
parser.add_argument(
'--ceph_conf',
type=str,
default="/etc/ceph/ceph.conf",
metavar='CEPHCONF',
help='CEPH configuration file used for cluster connect.')
parser.add_argument(
'--ts_limit',
type=int,
default=0,
help='Number of timestamps to process. Default is all.')
parser.add_argument('--pool',
type=str,
default=None,
metavar='POOL',
help='CEPH pool to use for object storage.')
parser.add_argument(
'--basename',
type=str,
default=None,
metavar='BASENAME',
help=
'Basename to use for object naming. Default is to use file start time.'
)
parser.add_argument(
'--redis',
type=str,
default=None,
help=
'Redis host to connect to as Telescope State. Default is to start a new local instance.'
)
parser.add_argument(
'--redis_port',
type=int,
default=6379,
help=
'Port to use when connecting to Redis instance (or creating a new one).'
)
parser.add_argument('--redis-only',
dest='redis_only',
action='store_true',
help='Only (re)build Redis DB - no object creation')
parser.add_argument(
'--obj_size',
type=int,
default=20,
help='Target obj size as a power of 2. Default: 2**20 (1 MB)')
args = parser.parse_args()
if args.file is None:
parser.error('argument --file is required')
if not rados:
logger.warning("No Ceph installation found - building Redis DB only")
args.redis_only = True
if not args.redis_only and args.pool is None:
parser.error('argument --pool is required')
if args.basename is None:
args.basename = args.file.split(".")[0]
return args
async def main():
katsdpservices.setup_logging()
logger = logging.getLogger("katsdpmetawriter")
katsdpservices.setup_restart()
parser = katsdpservices.ArgumentParser()
parser.add_argument('--rdb-path',
default="/var/kat/data",
metavar='RDBPATH',
help='Root in which to write RDB dumps')
parser.add_argument('--store-s3',
dest='store_s3',
default=False,
action='store_true',
help='Enable storage of RDB dumps in S3')
parser.add_argument(
'--access-key',
default="",
metavar='ACCESS',
help=
'S3 access key with write permission to the specified bucket [unauthenticated]'
)
parser.add_argument(
'--secret-key',
default="",
metavar='SECRET',
help='S3 secret key for the specified access key [unauthenticated]')
parser.add_argument('--s3-host',
default='localhost',
metavar='HOST',
help='S3 gateway host address [%(default)s]')
parser.add_argument('--s3-port',
default=7480,
metavar='PORT',
help='S3 gateway port [%(default)s]')
parser.add_argument('-p',
'--port',
type=int,
default=2049,
metavar='N',
help='KATCP host port [%(default)s]')
parser.add_argument('-a',
'--host',
default="",
metavar='HOST',
help='KATCP host address [all hosts]')
args = parser.parse_args()
if not os.path.exists(args.rdb_path):
logger.error("Specified RDB path, %s, does not exist.", args.rdb_path)
sys.exit(2)
botocore_dict = None
if args.store_s3:
botocore_dict = katsdpmetawriter.make_botocore_dict(args)
async with katsdpmetawriter.get_s3_connection(
botocore_dict, fail_on_boto=True) as s3_conn:
if s3_conn:
# we rebuild the connection each time we want to write a meta-data dump
logger.info("Successfully tested connection to S3 endpoint.")
else:
logger.warning(
"S3 endpoint %s:%s not available. Files will only be written locally.",
args.s3_host, args.s3_port)
else:
logger.info(
"Running in disk only mode. RDB dumps will not be written to S3")
telstate = await get_async_telstate(args.telstate_endpoint)
server = katsdpmetawriter.MetaWriterServer(args.host, args.port,
botocore_dict, args.rdb_path,
telstate)
logger.info("Started meta-data writer server.")
loop = asyncio.get_event_loop()
await server.start()
for sig in [signal.SIGINT, signal.SIGTERM]:
loop.add_signal_handler(sig, lambda: on_shutdown(loop, server))
await server.join()
telstate.backend.close()
await telstate.backend.wait_closed()
def setup(self) -> None:
self.parser = katsdpservices.ArgumentParser()
add_common_args(self.parser)
def parse_args(args=None, namespace=None):
"""Parse command-line arguments.
Any arguments are forwarded to :meth:`katsdpservices.ArgumentParser.parse_args`.
"""
defaults = SessionConfig('')
parser = katsdpservices.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--cbf-spead',
type=katsdptelstate.endpoint.endpoint_list_parser(7148),
default=':7148',
metavar='ENDPOINTS',
help=
('endpoints to listen for CBF SPEAD stream (including multicast IPs). '
'[<ip>[+<count>]][:port].'))
parser.add_argument('--stream-name',
type=str,
metavar='NAME',
help='Stream name for metadata in telstate')
parser.add_argument('--channels',
type=Range.parse,
metavar='A:B',
help='Output channels')
parser.add_argument('--log-level',
'-l',
type=str,
metavar='LEVEL',
default=None,
help='log level')
parser.add_argument('--file-base',
type=str,
metavar='DIR',
help='write HDF5 files in this directory if given')
parser.add_argument('--affinity',
type=spead2.parse_range_list,
metavar='CPU,CPU',
help='List of CPUs to which to bind threads')
parser.add_argument('--interface',
type=str,
help='Network interface for multicast subscription')
parser.add_argument('--direct-io',
action='store_true',
help='Use Direct I/O VFD for writing the file')
parser.add_argument('--ibv',
action='store_true',
help='Use libibverbs when possible')
parser.add_argument('--buffer-size',
type=int,
metavar='BYTES',
default=defaults.buffer_size,
help='Network buffer size [%(default)s]')
parser.add_argument('--max-packet',
type=int,
metavar='BYTES',
default=defaults.max_packet,
help='Maximum packet size (UDP payload) [%(default)s]')
parser.add_argument(
'--stats',
type=katsdptelstate.endpoint.endpoint_parser(7149),
metavar='ENDPOINT',
help='Send statistics to a signal display server at this address')
parser.add_argument(
'--stats-int-time',
type=float,
default=1.0,
metavar='SECONDS',
help='Interval between sending statistics to the signal displays')
parser.add_argument('--stats-interface',
type=str,
help='Network interface for signal display stream')
parser.add_aiomonitor_arguments()
parser.add_argument('--port',
'-p',
type=int,
default=2050,
help='katcp host port')
parser.add_argument('--host',
'-a',
type=str,
default='',
help='katcp host address')
args = parser.parse_args(args, namespace)
if args.affinity and len(args.affinity) < 2:
parser.error('At least 2 CPUs must be specified for --affinity')
if args.telstate is None:
parser.error('--telstate is required')
if args.stream_name is None:
parser.error('--stream-name is required')
return args