def main(argv: List[str]) -> None:
katsdpservices.setup_logging()
katsdpservices.setup_restart()
args = master_controller.parse_args(argv)
if args.log_level is not None:
logging.root.setLevel(args.log_level.upper())
if args.interface_mode:
logging.warning("Note: Running master controller in interface mode. "
"This allows testing of the interface only, "
"no actual command logic will be enacted.")
rewrite_gui_urls: Optional[Callable[[aiokatcp.Sensor], bytes]]
if args.haproxy:
rewrite_gui_urls = functools.partial(web.rewrite_gui_urls, args.external_url)
else:
rewrite_gui_urls = None
loop = asyncio.get_event_loop()
server = master_controller.DeviceServer(args, rewrite_gui_urls=rewrite_gui_urls)
for sig in [signal.SIGINT, signal.SIGTERM]:
loop.add_signal_handler(sig, functools.partial(handle_signal, server))
with katsdpservices.start_aiomonitor(loop, args, locals()):
loop.run_until_complete(async_main(server, args))
loop.close()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('dataset', type=str, help='Input dataset')
parser.add_argument('output_dir', type=str, help='Parent directory for output')
parser.add_argument('prefix', type=str, help='Prefix for output directories and filenames')
parser.add_argument('stream', type=str, help='Stream name for telescope state inputs')
parser.add_argument('--log-level', type=str, metavar='LEVEL',
help='Logging level [INFO]')
args = parser.parse_args()
katsdpservices.setup_logging()
if args.log_level is not None:
logging.getLogger().setLevel(args.log_level.upper())
dataset = katdal.open(args.dataset, chunk_store=None, upgrade_flags=False)
telstate = dataset.source.telstate.wrapped.root()
telstate = telstate.view(telstate.join(dataset.source.capture_block_id, args.stream))
output_dir = '{}_{}'.format(args.prefix, uuid.uuid4())
output_dir = os.path.join(args.output_dir, output_dir)
filename = args.prefix + '_report.html'
tmp_dir = output_dir + '.writing'
os.mkdir(tmp_dir)
try:
common_stats, target_stats = report.get_stats(dataset, telstate)
report.write_report(common_stats, target_stats, os.path.join(tmp_dir, filename))
report.write_metadata(dataset, common_stats, target_stats,
os.path.join(tmp_dir, 'metadata.json'))
os.rename(tmp_dir, output_dir)
except Exception:
# Make a best effort to clean up
shutil.rmtree(tmp_dir, ignore_errors=True)
raise
def main() -> None:
katsdpservices.setup_logging()
katsdpservices.setup_restart()
args = parse_args()
logger = logging.getLogger("katsdpcam2telstate")
loop = asyncio.get_event_loop()
client = Client(args, logger)
client.parse_streams()
with katsdpservices.start_aiomonitor(loop, args, locals()):
loop.run_until_complete(client.run())
def test_simple(self):
katsdpservices.setup_logging()
logging.debug('debug message')
logging.info('info message')
logging.warning('warning message\nwith\nnewlines')
self.assertRegex(
self.stderr.getvalue(),
re.compile(
"\\A2017-03-02T14:02:03.125Z - test_logging.py:\\d+ - INFO - info message\n"
"2017-03-02T14:02:03.125Z - test_logging.py:\\d+ - WARNING - warning message\n"
"with\n"
"newlines\n\\Z", re.M))
def test_log_level(self):
os.environ['KATSDP_LOG_LEVEL'] = 'debug'
katsdpservices.setup_logging()
logging.debug('debug message')
logging.info('info message')
logging.warning('warning message\nwith\nnewlines')
self.assertRegex(
self.stderr.getvalue(),
re.compile(
"\\A2017-03-02T14:02:03.125Z - test_logging.py:\\d+ - DEBUG - debug message\n"
"2017-03-02T14:02:03.125Z - test_logging.py:\\d+ - INFO - info message\n"
"2017-03-02T14:02:03.125Z - test_logging.py:\\d+ - WARNING - warning message\n"
"with\n"
"newlines\n\\Z", re.M))
def test_one_line(self):
os.environ['KATSDP_LOG_ONELINE'] = '1'
katsdpservices.setup_logging()
logging.debug('debug message')
logging.info('info message')
logging.warning('warning message\nwith\nnewlines')
# The \\\\ in the regex is a literal \: one level of escaping for
# Python, one for regex.
self.assertRegex(
self.stderr.getvalue(),
re.compile(
"\\A2017-03-02T14:02:03.125Z - test_logging.py:\\d+ - INFO - info message\n"
"2017-03-02T14:02:03.125Z - test_logging.py:\\d+ - WARNING - warning message\\\\ "
"with\\\\ "
"newlines\n\\Z", re.M))
async def main():
setup_logging()
opts = parse_opts()
telstate = opts.telstate
logging.info("Opening file %s", opts.file)
simdata = SimData.factory(opts.file, bchan=opts.bchan, echan=opts.echan,
n_substreams=opts.substreams)
async with simdata:
logging.info("Clearing telescope state")
telstate.clear()
logging.info("Setting values in telescope state")
simdata.setup_telstate(telstate)
def _test_gelf(self, localname, extra):
self.maxDiff = 4096 # Diff tends to be too big to display otherwise
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP)
with closing(sock):
sock.bind(('127.0.0.1', 0))
port = sock.getsockname()[1]
os.environ['KATSDP_LOG_GELF_ADDRESS'] = '127.0.0.1:{}'.format(port)
if localname:
os.environ['KATSDP_LOG_GELF_LOCALNAME'] = 'myhost'
if extra:
os.environ[
'KATSDP_LOG_GELF_EXTRA'] = '{"hello": "world", "number": 3}'
# Fake the container ID
container_id = "abcdef0123456789"
with mock.patch('katsdpservices.logging.docker_container_id',
return_value=container_id):
katsdpservices.setup_logging()
# exc_info=False is to test the fix for
# https://github.com/keeprocking/pygelf/issues/29
logging.info('info message', exc_info=False)
raw = sock.recv(4096)
raw = zlib.decompress(raw)
data = json.loads(raw.decode('utf-8'))
# This dictionary may need to be updated depending on the implementation
expected = {
"timestamp": self.time.return_value,
"version": "1.1",
"short_message": "info message",
"_logger_name": "katsdpservices.test.dummy",
"_file": mock.ANY,
"_line": mock.ANY,
"_func": "_test_gelf",
"_module": "test_logging",
"_docker.id": container_id,
"_timestamp_precise": "2017-03-02T14:02:03.125125Z",
"level": 6,
"host": "myhost" if localname else mock.ANY
}
if extra:
expected["_hello"] = "world"
expected["_number"] = 3
expected["_stack_info"] = None
self.assertEqual(data, expected)
async def main():
setup_logging()
opts = parse_opts()
logger.info("Use TS set up by sim_ts.py and run_cal.py scripts.")
telstate = opts.telstate
n_substreams = get_n_substreams(telstate)
simdata = SimData.factory(opts.file, opts.server, bchan=opts.bchan, echan=opts.echan,
n_substreams=n_substreams)
async with simdata:
logger.info("Issuing capture-init")
await simdata.capture_init()
logger.info("TX: start.")
await simdata.data_to_spead(telstate, opts.l0_spead, opts.l0_rate, max_scans=opts.max_scans,
interface=opts.l0_interface)
logger.info("TX: ended.")
logger.info("Issuing capture-done")
await simdata.capture_done()
def main():
parser = get_parser()
args = parser.parse_args(namespace=arguments.SmartNamespace())
katsdpservices.setup_logging()
if args.log_level is not None:
logger.setLevel(args.log_level.upper())
profiling.Profiler.set_profiler(profiling.FlamegraphProfiler())
with closing(
loader.load(args.input_file, args.input_option, args.start_channel,
args.stop_channel)) as dataset:
writer = Writer(args, dataset)
context = accel.create_some_context(interactive=False,
device_filter=lambda x: x.is_cuda)
queue = context.create_command_queue()
frontend.run(args, context, queue, dataset, writer)
# frontend.run modifies args.stop_channel in place, so even if it
# wasn't specified by the user it will now be valid.
writer.finalize(dataset, args.start_channel, args.stop_channel)
def main() -> None:
katsdpservices.setup_logging()
katsdpservices.setup_restart()
args = parse_args()
if args.log_level is not None:
logging.root.setLevel(args.log_level.upper())
if args.file_base is None and args.stats is None:
logging.warning(
'Neither --file-base nor --stats was given; nothing useful will happen'
)
if args.file_base is not None and not os.access(args.file_base, os.W_OK):
logging.error('Target directory (%s) is not writable', args.file_base)
sys.exit(1)
loop = asyncio.get_event_loop()
server = KatcpCaptureServer(args, loop)
loop.add_signal_handler(signal.SIGINT, lambda: on_shutdown(server))
loop.add_signal_handler(signal.SIGTERM, lambda: on_shutdown(server))
with katsdpservices.start_aiomonitor(loop, args, locals()):
loop.run_until_complete(server.start())
loop.run_until_complete(server.join())
loop.close()
upload_size = sum([product.upload_size() for product in upload_list])
upload_files = []
for product in upload_list:
product.update_state('TRANSFER_STARTED')
upload_files.extend(product.staged_for_transfer)
logger.debug("Uploading %.2f MB of data", (upload_size // 1e6))
uploader = Uploader(trawl_dir, boto_dict, upload_files)
uploader.upload()
failed_count = uploader.set_failed_tokens(solr_url)
logger.info(
f'A total of {failed_count} exceptions where encountered this cycle.')
return upload_size
if __name__ == "__main__":
katsdpservices.setup_logging()
logging.basicConfig(level=logging.INFO)
katsdpservices.setup_restart()
parser = OptionParser(usage="vis_trawler.py <trawl_directory>")
parser.add_option("--s3-host", default="localhost",
help="S3 gateway host address [default = %default]")
parser.add_option("--s3-port", type="int", default=7480,
help="S3 gateway port [default = %default]")
parser.add_option("--solr-url", default="http://kat-archive.kat.ac.za:8983/solr/kat_core",
help="Solr end point for metadata extraction [default = %default]")
(options, args) = parser.parse_args()
if len(args) < 1 or not os.path.isdir(args[0]):
print(__doc__)
sys.exit()
def main():
setup_logging()
parser = create_parser()
args = parser.parse_args()
# Open the observation
if (args.access_key is not None) != (args.secret_key is not None):
parser.error('--access-key and --secret-key must be used together')
if args.access_key is not None and args.token is not None:
parser.error('--access-key/--secret-key cannot be used with --token')
open_kwargs = {}
if args.access_key is not None:
open_kwargs['credentials'] = (args.access_key, args.secret_key)
elif args.token is not None:
open_kwargs['token'] = args.token
katdata = katdal.open(args.katdata, applycal='l1', **open_kwargs)
post_process_args(args, katdata)
uvblavg_args, mfimage_args, band = _infer_defaults_from_katdal(katdata)
# Get config defaults for uvblavg and mfimage and merge user supplied ones
uvblavg_parm_file = pjoin(CONFIG, f'uvblavg_MKAT_{band}.yaml')
log.info('UVBlAvg parameter file for %s-band: %s', band, uvblavg_parm_file)
mfimage_parm_file = pjoin(CONFIG, f'mfimage_MKAT_{band}.yaml')
log.info('MFImage parameter file for %s-band: %s', band, mfimage_parm_file)
user_uvblavg_args = get_and_merge_args(uvblavg_parm_file, args.uvblavg)
user_mfimage_args = get_and_merge_args(mfimage_parm_file, args.mfimage)
# Merge katdal defaults with user supplied defaults
recursive_merge(user_uvblavg_args, uvblavg_args)
recursive_merge(user_mfimage_args, mfimage_args)
# Get the default config.
dc = kc.get_config()
# Set up aipsdisk configuration from args.workdir
if args.workdir is not None:
aipsdirs = [(None,
pjoin(args.workdir, args.capture_block_id + '_aipsdisk'))]
else:
aipsdirs = dc['aipsdirs']
log.info('Using AIPS data area: %s', aipsdirs[0][1])
# Set up output configuration from args.outputdir
fitsdirs = dc['fitsdirs']
outputname = args.capture_block_id + OUTDIR_SEPARATOR + args.telstate_id + \
OUTDIR_SEPARATOR + START_TIME
outputdir = pjoin(args.outputdir, outputname)
# Set writing tag for duration of the pipeline
work_outputdir = outputdir + WRITE_TAG
# Append outputdir to fitsdirs
# NOTE: Pipeline is set up to always place its output in the
# highest numbered fits disk so we ensure that is the case
# here.
fitsdirs += [(None, work_outputdir)]
log.info('Using output data area: %s', outputdir)
kc.set_config(aipsdirs=aipsdirs, fitsdirs=fitsdirs)
setup_aips_disks()
# Add output_id and capture_block_id to configuration
kc.set_config(cfg=kc.get_config(),
output_id=args.output_id,
cb_id=args.capture_block_id)
# Set up telstate link then create
# a view based the capture block ID and output ID
telstate = TelescopeState(args.telstate)
view = telstate.join(args.capture_block_id, args.telstate_id)
ts_view = telstate.view(view)
katdal_select = args.select
katdal_select['nif'] = args.nif
# Create Continuum Pipeline
pipeline = pipeline_factory('online',
katdata,
ts_view,
katdal_select=katdal_select,
uvblavg_params=uvblavg_args,
mfimage_params=mfimage_args,
nvispio=args.nvispio)
# Execute it
metadata = pipeline.execute()
# Create QA products if images were created
if metadata:
make_pbeam_images(metadata, outputdir, WRITE_TAG)
make_qa_report(metadata, outputdir, WRITE_TAG)
organise_qa_output(metadata, outputdir, WRITE_TAG)
# Remove the writing tag from the output directory
os.rename(work_outputdir, outputdir)
else:
os.rmdir(work_outputdir)
async def main():
katsdpservices.setup_logging()
logger = logging.getLogger("katsdpmetawriter")
katsdpservices.setup_restart()
parser = katsdpservices.ArgumentParser()
parser.add_argument('--rdb-path',
default="/var/kat/data",
metavar='RDBPATH',
help='Root in which to write RDB dumps')
parser.add_argument('--store-s3',
dest='store_s3',
default=False,
action='store_true',
help='Enable storage of RDB dumps in S3')
parser.add_argument(
'--access-key',
default="",
metavar='ACCESS',
help=
'S3 access key with write permission to the specified bucket [unauthenticated]'
)
parser.add_argument(
'--secret-key',
default="",
metavar='SECRET',
help='S3 secret key for the specified access key [unauthenticated]')
parser.add_argument('--s3-host',
default='localhost',
metavar='HOST',
help='S3 gateway host address [%(default)s]')
parser.add_argument('--s3-port',
default=7480,
metavar='PORT',
help='S3 gateway port [%(default)s]')
parser.add_argument('-p',
'--port',
type=int,
default=2049,
metavar='N',
help='KATCP host port [%(default)s]')
parser.add_argument('-a',
'--host',
default="",
metavar='HOST',
help='KATCP host address [all hosts]')
args = parser.parse_args()
if not os.path.exists(args.rdb_path):
logger.error("Specified RDB path, %s, does not exist.", args.rdb_path)
sys.exit(2)
botocore_dict = None
if args.store_s3:
botocore_dict = katsdpmetawriter.make_botocore_dict(args)
async with katsdpmetawriter.get_s3_connection(
botocore_dict, fail_on_boto=True) as s3_conn:
if s3_conn:
# we rebuild the connection each time we want to write a meta-data dump
logger.info("Successfully tested connection to S3 endpoint.")
else:
logger.warning(
"S3 endpoint %s:%s not available. Files will only be written locally.",
args.s3_host, args.s3_port)
else:
logger.info(
"Running in disk only mode. RDB dumps will not be written to S3")
telstate = await get_async_telstate(args.telstate_endpoint)
server = katsdpmetawriter.MetaWriterServer(args.host, args.port,
botocore_dict, args.rdb_path,
telstate)
logger.info("Started meta-data writer server.")
loop = asyncio.get_event_loop()
await server.start()
for sig in [signal.SIGINT, signal.SIGTERM]:
loop.add_signal_handler(sig, lambda: on_shutdown(loop, server))
await server.join()
telstate.backend.close()
await telstate.backend.wait_closed()
def main() -> None:
args = get_arguments()
katsdpservices.setup_logging()
logging.root.setLevel(args.log_level.upper())
app = make_app(args)
web.run_app(app, host=args.bind, port=args.port)
def create_parser():
formatter_class = argparse.ArgumentDefaultsHelpFormatter
parser = argparse.ArgumentParser(formatter_class=formatter_class)
parser.add_argument("-a",
"--aipsdisks",
default=None,
type=lambda s: [(None, ds.strip())
for ds in s.split(',')],
help="Comma separated list of paths to aipsdisks.")
parser.add_argument("-f",
"--fitsdisks",
default=None,
type=lambda s: [(None, ds.strip())
for ds in s.split(',')],
help="Comma separated list of paths to fitsdisks.")
return parser
setup_logging()
args = create_parser().parse_args()
kc.set_config(aipsdirs=args.aipsdisks, fitsdirs=args.fitsdisks)
setup_aips_disks()
rewrite_dadevs()
rewrite_netsp()
link_obit_data()
def main() -> None:
parser, args = parse_args()
prepare_env(args)
katsdpservices.setup_logging()
katsdpservices.setup_restart()
if args.log_level is not None:
logging.root.setLevel(args.log_level.upper())
logger = logging.getLogger('katsdpcontroller')
logger.info("Starting SDP product controller...")
logger.info('katcp: %s:%d', args.host, args.port)
logger.info('http: %s', args.http_url)
master_controller = aiokatcp.Client(args.master_controller.host,
args.master_controller.port)
image_lookup = product_controller.KatcpImageLookup(master_controller)
try:
image_resolver_factory = make_image_resolver_factory(
image_lookup, args)
except ValueError as exc:
parser.error(str(exc))
framework_info = addict.Dict()
framework_info.user = args.user
framework_info.name = args.subarray_product_id
framework_info.checkpoint = True
framework_info.principal = args.principal
framework_info.roles = [args.realtime_role, args.batch_role]
framework_info.capabilities = [{
'type': 'MULTI_ROLE'
}, {
'type': 'TASK_KILLING_STATE'
}]
loop = asyncio.get_event_loop()
sched = scheduler.Scheduler(
args.realtime_role,
args.host,
args.http_port,
args.http_url,
task_stats=product_controller.TaskStats(),
runner_kwargs=dict(access_log_class=web_utils.AccessLogger))
sched.app.router.add_get('/metrics', web_utils.prometheus_handler)
sched.app.router.add_get('/health', web_utils.health_handler)
driver = pymesos.MesosSchedulerDriver(sched,
framework_info,
args.mesos_master,
use_addict=True,
implicit_acknowledgements=False)
sched.set_driver(driver)
driver.start()
dashboard_path = f'/gui/{args.subarray_product_id}/product/dashboard/'
dashboard_url: Optional[str] = args.dashboard_url
if args.dashboard_port != 0 and dashboard_url is None:
dashboard_url = str(
yarl.URL.build(scheme='http',
host=args.external_hostname,
port=args.dashboard_port,
path=dashboard_path))
server = product_controller.DeviceServer(
args.host,
args.port,
master_controller,
args.subarray_product_id,
sched,
batch_role=args.batch_role,
interface_mode=False,
localhost=args.localhost,
image_resolver_factory=image_resolver_factory,
s3_config=args.s3_config if args.s3_config is not None else {},
graph_dir=args.write_graphs,
dashboard_url=dashboard_url)
if args.dashboard_port != 0:
init_dashboard(server, args, dashboard_path)
with katsdpservices.start_aiomonitor(loop, args, locals()):
loop.run_until_complete(run(sched, server))
loop.close()
