def main(argv: List[str]) -> None:
katsdpservices.setup_logging()
katsdpservices.setup_restart()
args = master_controller.parse_args(argv)
if args.log_level is not None:
logging.root.setLevel(args.log_level.upper())
if args.interface_mode:
logging.warning("Note: Running master controller in interface mode. "
"This allows testing of the interface only, "
"no actual command logic will be enacted.")
rewrite_gui_urls: Optional[Callable[[aiokatcp.Sensor], bytes]]
if args.haproxy:
rewrite_gui_urls = functools.partial(web.rewrite_gui_urls, args.external_url)
else:
rewrite_gui_urls = None
loop = asyncio.get_event_loop()
server = master_controller.DeviceServer(args, rewrite_gui_urls=rewrite_gui_urls)
for sig in [signal.SIGINT, signal.SIGTERM]:
loop.add_signal_handler(sig, functools.partial(handle_signal, server))
with katsdpservices.start_aiomonitor(loop, args, locals()):
loop.run_until_complete(async_main(server, args))
loop.close()
def test_restart_signal_with_true_callback(self):
callback = mock.MagicMock()
callback.return_value = True
katsdpservices.setup_restart(restart_callback=callback)
os.kill(os.getpid(), signal.SIGHUP)
# Give time for asynchronous handling
time.sleep(0.01)
callback.assert_called_once_with()
self.execlp.assert_not_called()
def main() -> None:
katsdpservices.setup_logging()
katsdpservices.setup_restart()
args = parse_args()
logger = logging.getLogger("katsdpcam2telstate")
loop = asyncio.get_event_loop()
client = Client(args, logger)
client.parse_streams()
with katsdpservices.start_aiomonitor(loop, args, locals()):
loop.run_until_complete(client.run())
def main() -> None:
katsdpservices.setup_logging()
katsdpservices.setup_restart()
args = parse_args()
if args.log_level is not None:
logging.root.setLevel(args.log_level.upper())
if args.file_base is None and args.stats is None:
logging.warning(
'Neither --file-base nor --stats was given; nothing useful will happen'
)
if args.file_base is not None and not os.access(args.file_base, os.W_OK):
logging.error('Target directory (%s) is not writable', args.file_base)
sys.exit(1)
loop = asyncio.get_event_loop()
server = KatcpCaptureServer(args, loop)
loop.add_signal_handler(signal.SIGINT, lambda: on_shutdown(server))
loop.add_signal_handler(signal.SIGTERM, lambda: on_shutdown(server))
with katsdpservices.start_aiomonitor(loop, args, locals()):
loop.run_until_complete(server.start())
loop.run_until_complete(server.join())
loop.close()
for product in upload_list:
product.update_state('TRANSFER_STARTED')
upload_files.extend(product.staged_for_transfer)
logger.debug("Uploading %.2f MB of data", (upload_size // 1e6))
uploader = Uploader(trawl_dir, boto_dict, upload_files)
uploader.upload()
failed_count = uploader.set_failed_tokens(solr_url)
logger.info(
f'A total of {failed_count} exceptions where encountered this cycle.')
return upload_size
if __name__ == "__main__":
katsdpservices.setup_logging()
logging.basicConfig(level=logging.INFO)
katsdpservices.setup_restart()
parser = OptionParser(usage="vis_trawler.py <trawl_directory>")
parser.add_option("--s3-host", default="localhost",
help="S3 gateway host address [default = %default]")
parser.add_option("--s3-port", type="int", default=7480,
help="S3 gateway port [default = %default]")
parser.add_option("--solr-url", default="http://kat-archive.kat.ac.za:8983/solr/kat_core",
help="Solr end point for metadata extraction [default = %default]")
(options, args) = parser.parse_args()
if len(args) < 1 or not os.path.isdir(args[0]):
print(__doc__)
sys.exit()
boto_dict = make_boto_dict(options)
def test_restart_signal(self):
katsdpservices.setup_restart()
os.kill(os.getpid(), signal.SIGHUP)
# Give time for asynchronous handling
time.sleep(0.01)
self.assertEqual(1, len(self.execlp.mock_calls))
async def main():
katsdpservices.setup_logging()
logger = logging.getLogger("katsdpmetawriter")
katsdpservices.setup_restart()
parser = katsdpservices.ArgumentParser()
parser.add_argument('--rdb-path',
default="/var/kat/data",
metavar='RDBPATH',
help='Root in which to write RDB dumps')
parser.add_argument('--store-s3',
dest='store_s3',
default=False,
action='store_true',
help='Enable storage of RDB dumps in S3')
parser.add_argument(
'--access-key',
default="",
metavar='ACCESS',
help=
'S3 access key with write permission to the specified bucket [unauthenticated]'
)
parser.add_argument(
'--secret-key',
default="",
metavar='SECRET',
help='S3 secret key for the specified access key [unauthenticated]')
parser.add_argument('--s3-host',
default='localhost',
metavar='HOST',
help='S3 gateway host address [%(default)s]')
parser.add_argument('--s3-port',
default=7480,
metavar='PORT',
help='S3 gateway port [%(default)s]')
parser.add_argument('-p',
'--port',
type=int,
default=2049,
metavar='N',
help='KATCP host port [%(default)s]')
parser.add_argument('-a',
'--host',
default="",
metavar='HOST',
help='KATCP host address [all hosts]')
args = parser.parse_args()
if not os.path.exists(args.rdb_path):
logger.error("Specified RDB path, %s, does not exist.", args.rdb_path)
sys.exit(2)
botocore_dict = None
if args.store_s3:
botocore_dict = katsdpmetawriter.make_botocore_dict(args)
async with katsdpmetawriter.get_s3_connection(
botocore_dict, fail_on_boto=True) as s3_conn:
if s3_conn:
# we rebuild the connection each time we want to write a meta-data dump
logger.info("Successfully tested connection to S3 endpoint.")
else:
logger.warning(
"S3 endpoint %s:%s not available. Files will only be written locally.",
args.s3_host, args.s3_port)
else:
logger.info(
"Running in disk only mode. RDB dumps will not be written to S3")
telstate = await get_async_telstate(args.telstate_endpoint)
server = katsdpmetawriter.MetaWriterServer(args.host, args.port,
botocore_dict, args.rdb_path,
telstate)
logger.info("Started meta-data writer server.")
loop = asyncio.get_event_loop()
await server.start()
for sig in [signal.SIGINT, signal.SIGTERM]:
loop.add_signal_handler(sig, lambda: on_shutdown(loop, server))
await server.join()
telstate.backend.close()
await telstate.backend.wait_closed()
def main() -> None:
parser, args = parse_args()
prepare_env(args)
katsdpservices.setup_logging()
katsdpservices.setup_restart()
if args.log_level is not None:
logging.root.setLevel(args.log_level.upper())
logger = logging.getLogger('katsdpcontroller')
logger.info("Starting SDP product controller...")
logger.info('katcp: %s:%d', args.host, args.port)
logger.info('http: %s', args.http_url)
master_controller = aiokatcp.Client(args.master_controller.host,
args.master_controller.port)
image_lookup = product_controller.KatcpImageLookup(master_controller)
try:
image_resolver_factory = make_image_resolver_factory(
image_lookup, args)
except ValueError as exc:
parser.error(str(exc))
framework_info = addict.Dict()
framework_info.user = args.user
framework_info.name = args.subarray_product_id
framework_info.checkpoint = True
framework_info.principal = args.principal
framework_info.roles = [args.realtime_role, args.batch_role]
framework_info.capabilities = [{
'type': 'MULTI_ROLE'
}, {
'type': 'TASK_KILLING_STATE'
}]
loop = asyncio.get_event_loop()
sched = scheduler.Scheduler(
args.realtime_role,
args.host,
args.http_port,
args.http_url,
task_stats=product_controller.TaskStats(),
runner_kwargs=dict(access_log_class=web_utils.AccessLogger))
sched.app.router.add_get('/metrics', web_utils.prometheus_handler)
sched.app.router.add_get('/health', web_utils.health_handler)
driver = pymesos.MesosSchedulerDriver(sched,
framework_info,
args.mesos_master,
use_addict=True,
implicit_acknowledgements=False)
sched.set_driver(driver)
driver.start()
dashboard_path = f'/gui/{args.subarray_product_id}/product/dashboard/'
dashboard_url: Optional[str] = args.dashboard_url
if args.dashboard_port != 0 and dashboard_url is None:
dashboard_url = str(
yarl.URL.build(scheme='http',
host=args.external_hostname,
port=args.dashboard_port,
path=dashboard_path))
server = product_controller.DeviceServer(
args.host,
args.port,
master_controller,
args.subarray_product_id,
sched,
batch_role=args.batch_role,
interface_mode=False,
localhost=args.localhost,
image_resolver_factory=image_resolver_factory,
s3_config=args.s3_config if args.s3_config is not None else {},
graph_dir=args.write_graphs,
dashboard_url=dashboard_url)
if args.dashboard_port != 0:
init_dashboard(server, args, dashboard_path)
with katsdpservices.start_aiomonitor(loop, args, locals()):
loop.run_until_complete(run(sched, server))
loop.close()