async def add_module(request): """Creates a new routing rule and forwards the user to the rule edit page.""" try: config.read_config() except: return PlainTextResponse( "Configuration is being updated. Try again in a minute.") form = dict(await request.form()) name = form.get("name", "") if name in config.mercure["modules"]: return PlainTextResponse("Name already exists.") config.mercure["modules"][name] = { "url": form.get("url", ""), "docker_tag": form.get("docker_tag", None) } try: config.save_config() except: return PlainTextResponse( "ERROR: Unable to write configuration. Try again.") # logger.info(f'Created rule {newrule}') # monitor.send_webgui_event(monitor.w_events.RULE_CREATE, request.user.display_name, newrule) return RedirectResponse(url="/modules/", status_code=303)
async def add_rule(request): """Creates a new routing rule and forwards the user to the rule edit page.""" try: config.read_config() except: return PlainTextResponse( 'Configuration is being updated. Try again in a minute.') form = dict(await request.form()) newrule = form.get("name", "") if newrule in config.mercure["rules"]: return PlainTextResponse('Rule already exists.') config.mercure["rules"][newrule] = {"rule": "False"} try: config.save_config() except: return PlainTextResponse( 'ERROR: Unable to write configuration. Try again.') logger.info(f'Created rule {newrule}') monitor.send_webgui_event(monitor.w_events.RULE_CREATE, request.user.display_name, newrule) return RedirectResponse(url='/rules/edit/' + newrule, status_code=303)
async def show_queues(request): """Shows all installed modules""" try: config.read_config() except: return PlainTextResponse( 'Configuration is being updated. Try again in a minute.') processing_suspended = False processing_halt_file = Path(config.mercure['processing_folder'] + '/' + mercure_names.HALT) if processing_halt_file.exists(): processing_suspended = True routing_suspended = False routing_halt_file = Path(config.mercure['outgoing_folder'] + '/' + mercure_names.HALT) if routing_halt_file.exists(): routing_suspended = True template = "queue.html" context = { "request": request, "mercure_version": mercure_defs.VERSION, "page": "queue", "processing_suspended": processing_suspended, "routing_suspended": routing_suspended } context.update(get_user_information(request)) return templates.TemplateResponse(template, context)
def clean(args): """ Main entry function. """ if helper.is_terminated(): return helper.g_log("events.run", 1) try: config.read_config() except Exception: logger.exception("Unable to read configuration. Skipping processing.") monitor.send_event( monitor.h_events.CONFIG_UPDATE, monitor.severity.WARNING, "Unable to read configuration (possibly locked)", ) return # TODO: Adaptively reduce the retention time if the disk space is running low if _is_offpeak( config.mercure["offpeak_start"], config.mercure["offpeak_end"], datetime.now().time(), ): success_folder = config.mercure[mercure_folders.SUCCESS] discard_folder = config.mercure[mercure_folders.DISCARD] retention = timedelta(seconds=config.mercure["retention"]) clean_dir(success_folder, retention) clean_dir(discard_folder, retention)
async def show_queues_status(request): try: config.read_config() except: return PlainTextResponse( 'Configuration is being updated. Try again in a minute.') processing_suspended = False processing_halt_file = Path(config.mercure['processing_folder'] + '/HALT') if processing_halt_file.exists(): processing_suspended = True routing_suspended = False routing_halt_file = Path(config.mercure['outgoing_folder'] + '/HALT') if routing_halt_file.exists(): routing_suspended = True processing_status = "Idle" routing_status = "Idle" if (processing_suspended): processing_status = "Halted" if (routing_suspended): routing_status = "Halted" queue_status = { "processing_status": processing_status, "processing_suspended": str(processing_suspended), "routing_status": routing_status, "routing_suspended": str(routing_suspended) } return JSONResponse(queue_status)
async def show_modules(request): """Shows all installed modules""" try: config.read_config() except: return PlainTextResponse( "Configuration is being updated. Try again in a minute.") used_modules = {} for rule in config.mercure["rules"]: used_module = config.mercure["rules"][rule].get( "processing_module", "NONE") used_modules[used_module] = rule template = "modules.html" context = { "request": request, "mercure_version": mercure_defs.VERSION, "page": "modules", "modules": config.mercure["modules"], "used_modules": used_modules } context.update(get_user_information(request)) return templates.TemplateResponse(template, context)
async def set_queues_status(request): try: config.read_config() except: return PlainTextResponse( 'Configuration is being updated. Try again in a minute.') processing_halt_file = Path(config.mercure['processing_folder'] + '/' + mercure_names.HALT) routing_halt_file = Path(config.mercure['outgoing_folder'] + '/' + mercure_names.HALT) form = dict(await request.form()) print(form) try: if (form.get("suspend_processing", "false") == "true"): processing_halt_file.touch() else: processing_halt_file.unlink() except: pass try: if (form.get("suspend_routing", "false") == "true"): routing_halt_file.touch() else: routing_halt_file.unlink() except: pass return JSONResponse({'result': 'OK'})
def dispatch(args): """ Main entry function. """ if helper.is_terminated(): return helper.g_log("events.run", 1) try: config.read_config() except Exception: logger.exception("Unable to read configuration. Skipping processing.") monitor.send_event( monitor.h_events.CONFIG_UPDATE, monitor.severity.WARNING, "Unable to read configuration (possibly locked)", ) return success_folder = Path(config.mercure[mercure_folders.SUCCESS]) error_folder = Path(config.mercure[mercure_folders.ERROR]) retry_max = config.mercure["retry_max"] retry_delay = config.mercure["retry_delay"] # TODO: Sort list so that the oldest DICOMs get dispatched first with os.scandir(config.mercure[mercure_folders.OUTGOING]) as it: for entry in it: if entry.is_dir() and not has_been_send(entry.path) and is_ready_for_sending(entry.path): logger.info(f"Sending folder {entry.path}") execute(Path(entry.path), success_folder, error_folder, retry_max, retry_delay) # If termination is requested, stop processing series after the # active one has been completed if helper.is_terminated(): break
async def show_jobs_processing(request): try: config.read_config() except: return PlainTextResponse( 'Configuration is being updated. Try again in a minute.') job_list = {} job_list["1234-1234-1234-1234"] = { "Module": "Test", "ACC": "ACC1234", "MRN": "MRN1234", "Status": "Processing" } job_list["1334-1244-2234-1233"] = { "Module": "Anonymizer", "ACC": "ACC1234", "MRN": "MRN1234", "Status": "Scheduled" } job_list["4234-1234-1434-1234"] = { "Module": "Anonymizer", "ACC": "ACC1234", "MRN": "MRN1234", "Status": "Scheduled" } return JSONResponse(job_list)
async def add_target(request): """Creates a new target.""" try: config.read_config() except: return PlainTextResponse( 'Configuration is being updated. Try again in a minute.') form = dict(await request.form()) newtarget = form.get("name", "") if newtarget in config.mercure["targets"]: return PlainTextResponse('Target already exists.') config.mercure["targets"][newtarget] = {"ip": "", "port": ""} try: config.save_config() except: return PlainTextResponse( 'ERROR: Unable to write configuration. Try again.') logger.info(f'Created target {newtarget}') monitor.send_webgui_event(monitor.w_events.TARGET_CREATE, request.user.display_name, newtarget) return RedirectResponse(url='/targets/edit/' + newtarget, status_code=303)
async def targets_test_post(request): """Tests the connectivity of the given target by executing ping and c-echo requests.""" try: config.read_config() except: return PlainTextResponse("Configuration is being updated. Try again in a minute.") testtarget = request.path_params["target"] ping_response = "False" cecho_response = "False" target_ip = "" target_port = "" target_aec = "ANY-SCP" target_aet = "ECHOSCU" try: target_ip = config.mercure["targets"][testtarget]["ip"] target_port = config.mercure["targets"][testtarget]["port"] target_aec = config.mercure["targets"][testtarget]["aet_target"] target_aet = config.mercure["targets"][testtarget]["aet_source"] except: pass logger.info(f"Testing target {testtarget}") if (target_ip) and (target_port): if (await async_run("ping -w 1 -c 1 " + target_ip))[0] == 0: ping_response = "True" # Only test for c-echo if the ping was successful if (await async_run("echoscu -to 10 -aec " + target_aec + " -aet " + target_aet + " " + target_ip + " " + target_port))[0] == 0: cecho_response = "True" return JSONResponse('{"ping": "' + ping_response + '", "c-echo": "' + cecho_response + '" }')
async def targes_edit_post(request): """Updates the given target using the form values posted with the request.""" try: config.read_config() except: return PlainTextResponse( 'Configuration is being updated. Try again in a minute.') edittarget = request.path_params["target"] form = dict(await request.form()) if not edittarget in config.mercure["targets"]: return PlainTextResponse('Target does not exist anymore.') config.mercure["targets"][edittarget]["ip"] = form["ip"] config.mercure["targets"][edittarget]["port"] = form["port"] config.mercure["targets"][edittarget]["aet_target"] = form["aet_target"] config.mercure["targets"][edittarget]["aet_source"] = form["aet_source"] config.mercure["targets"][edittarget]["contact"] = form["contact"] try: config.save_config() except: return PlainTextResponse( 'ERROR: Unable to write configuration. Try again.') logger.info(f'Edited target {edittarget}') monitor.send_webgui_event(monitor.w_events.TARGET_EDIT, request.user.display_name, edittarget) return RedirectResponse(url='/targets', status_code=303)
async def rules_edit_post(request): """Updates the settings for the given routing rule.""" try: config.read_config() except: return PlainTextResponse( "Configuration is being updated. Try again in a minute.") editrule = request.path_params["rule"] form = dict(await request.form()) if not editrule in config.mercure["rules"]: return PlainTextResponse("Rule does not exist anymore.") config.mercure["rules"][editrule]["rule"] = form.get("rule", "False") config.mercure["rules"][editrule]["target"] = form.get("target", "") config.mercure["rules"][editrule]["disabled"] = form.get( "status_disabled", "False") config.mercure["rules"][editrule]["fallback"] = form.get( "status_fallback", "False") config.mercure["rules"][editrule]["contact"] = form.get("contact", "") config.mercure["rules"][editrule]["comment"] = form.get("comment", "") config.mercure["rules"][editrule]["tags"] = form.get("tags", "") config.mercure["rules"][editrule]["action"] = form.get("action", "route") config.mercure["rules"][editrule]["action_trigger"] = form.get( "action_trigger", "series") config.mercure["rules"][editrule]["study_trigger_condition"] = form.get( "study_trigger_condition", "timeout") config.mercure["rules"][editrule]["study_trigger_series"] = form.get( "study_trigger_series", "") config.mercure["rules"][editrule]["priority"] = form.get( "priority", "normal") config.mercure["rules"][editrule]["processing_module"] = form.get( "processing_module", "") config.mercure["rules"][editrule]["processing_settings"] = form.get( "processing_settings", "") config.mercure["rules"][editrule]["notification_webhook"] = form.get( "notification_webhook", "") config.mercure["rules"][editrule]["notification_payload"] = form.get( "notification_payload", "") config.mercure["rules"][editrule][ "notification_trigger_reception"] = form.get( "notification_trigger_reception", "False") config.mercure["rules"][editrule][ "notification_trigger_completion"] = form.get( "notification_trigger_completion", "False") config.mercure["rules"][editrule]["notification_trigger_error"] = form.get( "notification_trigger_error", "False") try: config.save_config() except: return PlainTextResponse( "ERROR: Unable to write configuration. Try again.") logger.info(f"Edited rule {editrule}") monitor.send_webgui_event(monitor.w_events.RULE_EDIT, request.user.display_name, editrule) return RedirectResponse(url="/rules", status_code=303)
async def login(request): """Shows the login page.""" try: config.read_config() except: return PlainTextResponse("Error reading configuration file.") request.session.clear() template = "login.html" context = {"request": request, "mercure_version": mercure_defs.VERSION, "appliance_name": config.mercure.get("appliance_name", "master")} return templates.TemplateResponse(template, context)
async def show_rules(request): """Show all defined routing rules. Can be executed by all logged-in users.""" try: config.read_config() except: return PlainTextResponse("Configuration is being updated. Try again in a minute.") template = "rules.html" context = {"request": request, "mercure_version": mercure_defs.VERSION, "page": "rules", "rules": config.mercure["rules"]} context.update(get_user_information(request)) return templates.TemplateResponse(template, context)
def init(): conf = config.read_config("config_client3.json") globals().update(conf) global server_address server_address = [(host, port1), (host, port2), (host, port3)] sudp = None stcp = [[1, socket.socket(socket.AF_INET, socket.SOCK_STREAM), False, 'b'], [2, socket.socket(socket.AF_INET, socket.SOCK_STREAM), False, 'b'], [3, socket.socket(socket.AF_INET, socket.SOCK_STREAM), False, 'b']] # connect to server for i in range(len(stcp)): stcp[i][1].connect(server_address[i]) stcp[i][2] = True print("connecting to {} port {}".format(server_address[i][0], server_address[i][1])) logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) logging.info('Socket(%s:%i) connected', host, port1) logging.info('Socket(%s:%i) connected', host, port2) logging.info('Socket(%s:%i) connected', host, port3) logging.info('CID: %i, SEQ: %i', cid, seq) return sudp, stcp
def update_task_by_uuid(config_path, task_uuid, miner_fid, csv): config = read_config(config_path) api_url = config['main']['api_url'] api_key = config['main']['api_key'] access_token = config['main']['access_token'] client = SwanClient(api_url, api_key, access_token) client.update_task_by_uuid(task_uuid, miner_fid, csv)
def parse_args(): """ commandline args parsing and config reading """ parser = argparse.ArgumentParser() parser.add_argument("event_dir", help="Event directory - place to read config and store images") parser.add_argument("-d", "--dummy", help="Force dummy camera instead of GPhoto interface", action="store_true") parser.add_argument("-D", "--debug", help="Use debug configuration for easier development", action="store_true") parser.add_argument("-f", "--fullscreen", help="Force fullscreen mode", action="store_true") parser.add_argument("-u", "--upload", help="Force images upload to Tumblr", action="store_true") parser.add_argument("-s", "--setup", help="Use this mode to setup the camera", action="store_true") args = parser.parse_args() conf = config.read_config(args.event_dir) conf['event_dir'] = args.event_dir if args.dummy: conf['camera']['driver'] = "DummyCamera" conf['display']['fullscreen'] |= args.fullscreen conf['control']['save_path'] = args.event_dir conf['upload']['enabled'] |= args.upload # NOTE: also the first session will be the "setting up" one if args.setup: conf['control']['idle_secs'] = 360 conf['devices']['lights_default'] = conf['devices']['lights_full'] return conf
def send_deals(config_path, miner_id, task_name=None, metadata_csv_path=None, deal_list=None, task_uuid=None): config = read_config(config_path) from_wallet = config['sender']['wallet'] max_price = config['sender']['max_price'] is_verified = config['sender']['is_verified'] epoch_interval_hours = config['sender']['start_epoch_hours'] output_dir = config['sender']['output_dir'] deal_config = DealConfig(miner_id, from_wallet, max_price, is_verified, epoch_interval_hours) if deal_list: return send_deals_to_miner(deal_config, output_dir, task_name=task_name, deal_list=deal_list, task_uuid=task_uuid) elif metadata_csv_path: return send_deals_to_miner(deal_config, output_dir, csv_file_path=metadata_csv_path, task_uuid=task_uuid) else: logging.error("no valid deal list or metadata_csv provided")
async def targets_edit(request): """Shows the edit page for the given target.""" try: config.read_config() except: return PlainTextResponse("Configuration is being updated. Try again in a minute.") edittarget = request.path_params["target"] if not edittarget in config.mercure["targets"]: return RedirectResponse(url="/targets", status_code=303) template = "targets_edit.html" context = {"request": request, "mercure_version": mercure_defs.VERSION, "page": "targets", "targets": config.mercure["targets"], "edittarget": edittarget} context.update(get_user_information(request)) return templates.TemplateResponse(template, context)
async def show_targets(request): """Shows all configured targets.""" try: config.read_config() except: return PlainTextResponse("Configuration is being updated. Try again in a minute.") used_targets = {} for rule in config.mercure["rules"]: used_target = config.mercure["rules"][rule].get("target", "NONE") used_targets[used_target] = rule template = "targets.html" context = {"request": request, "mercure_version": mercure_defs.VERSION, "page": "targets", "targets": config.mercure["targets"], "used_targets": used_targets} context.update(get_user_information(request)) return templates.TemplateResponse(template, context)
async def edit_module(request): """Creates a new routing rule and forwards the user to the rule edit page.""" try: config.read_config() except: return PlainTextResponse( "Configuration is being updated. Try again in a minute.") form = dict(await request.form()) name = request.path_params["module"] if name not in config.mercure["modules"]: return PlainTextResponse( "Invalid module name - perhaps it was deleted?") return await save_module(form, name)
async def add_module(request): """Creates a new routing rule and forwards the user to the rule edit page.""" try: config.read_config() except: return PlainTextResponse( "Configuration is being updated. Try again in a minute.") form = dict(await request.form()) name = form.get("name", "") if name in config.mercure["modules"]: return PlainTextResponse("Name already exists.") # logger.info(f'Created rule {name}') # monitor.send_webgui_event(monitor.w_events.RULE_CREATE, request.user.display_name, name) return await save_module(form, name)
def init(): conf = config.read_config("config_repmgn.json") globals().update(conf) sudp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sudp.bind((mask, port)) logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) logging.info('Socket(%s:%i) binded', mask, port) return sudp
async def edit_module(request): """Shows all installed modules""" module = request.path_params["module"] try: config.read_config() except: return PlainTextResponse( "Configuration is being updated. Try again in a minute.") template = "modules_edit.html" context = { "request": request, "mercure_version": mercure_defs.VERSION, "page": "modules", "module": config.mercure["modules"][module], "module_name": module } context.update(get_user_information(request)) return templates.TemplateResponse(template, context)
async def rules_delete_post(request): """Deletes the given routing rule""" try: config.read_config() except: return PlainTextResponse("Configuration is being updated. Try again in a minute.") deleterule = request.path_params["rule"] if deleterule in config.mercure["rules"]: del config.mercure["rules"][deleterule] try: config.save_config() except: return PlainTextResponse("ERROR: Unable to write configuration. Try again.") logger.info(f"Deleted rule {deleterule}") monitor.send_webgui_event(monitor.w_events.RULE_DELETE, request.user.display_name, deleterule) return RedirectResponse(url="/rules", status_code=303)
async def configuration(request): """Shows the current configuration of the mercure appliance.""" try: config.read_config() except: return PlainTextResponse('Error reading configuration file.') template = "configuration.html" config_edited = int(request.query_params.get("edited", 0)) os_info = distro.linux_distribution() os_string = f"{os_info[0]} Version {os_info[1]} ({os_info[2]})" context = { "request": request, "mercure_version": mercure_defs.VERSION, "page": "configuration", "config": config.mercure, "os_string": os_string, "config_edited": config_edited } context.update(get_user_information(request)) return templates.TemplateResponse(template, context)
async def delete_module(request): """Creates a new routing rule and forwards the user to the rule edit page.""" try: config.read_config() except: return PlainTextResponse( "Configuration is being updated. Try again in a minute.") name = request.path_params["module"] if name in config.mercure["modules"]: del config.mercure["modules"][name] try: config.save_config() except: return PlainTextResponse( "ERROR: Unable to write configuration. Try again.") # logger.info(f'Created rule {newrule}') # monitor.send_webgui_event(monitor.w_events.RULE_CREATE, request.user.display_name, newrule) return RedirectResponse(url="/modules/", status_code=303)
async def targets_delete_post(request): """Deletes the given target.""" try: config.read_config() except: return PlainTextResponse("Configuration is being updated. Try again in a minute.") deletetarget = request.path_params["target"] if deletetarget in config.mercure["targets"]: del config.mercure["targets"][deletetarget] try: config.save_config() except: return PlainTextResponse("ERROR: Unable to write configuration. Try again.") logger.info(f"Deleted target {deletetarget}") monitor.send_webgui_event(monitor.w_events.TARGET_DELETE, request.user.display_name, deletetarget) return RedirectResponse(url="/targets", status_code=303)
async def users_delete_post(request): """Deletes the given users.""" try: config.read_config() except: return PlainTextResponse("Configuration is being updated. Try again in a minute.") deleteuser = request.path_params["user"] if deleteuser in users.users_list: del users.users_list[deleteuser] try: users.save_users() except: return PlainTextResponse("ERROR: Unable to write user list. Try again.") logger.info(f"Deleted user {deleteuser}") monitor.send_webgui_event(monitor.w_events.USER_DELETE, request.user.display_name, deleteuser) return RedirectResponse(url="/users", status_code=303)
async def configuration_edit_post(request): """Updates the configuration after post from editor""" form = dict(await request.form()) editor_json = form.get("editor", "{}") try: validated_json = json.loads(editor_json) except ValueError: return PlainTextResponse("Invalid JSON data transferred.") try: config.write_configfile(validated_json) config.read_config() except ValueError: return PlainTextResponse("Unable to write config file. Might be locked.") logger.info(f"Updates mercure configuration file.") monitor.send_webgui_event(monitor.w_events.CONFIG_EDIT, request.user.display_name, "") return RedirectResponse(url="/configuration?edited=1", status_code=303)
def parse_args(): """ commandline args parsing and config reading """ parser = argparse.ArgumentParser() parser.add_argument("event_dir", help="Event directory - place to read config") parser.add_argument("-w", "--webserver", help="Start webserver to serve videos locally", action="store_true") args = parser.parse_args() conf = config.read_config(args.event_dir, default_config=config.DEFAULT_CONFIG_FILE_VIDEO) conf['event_dir'] = args.event_dir conf['webserver']['enabled'] |= args.webserver return conf
def test_multiple_files(self): """ Test how several config files are merged in one """ res = config.read_config("test/resources/*.json") def expect(field, condition): assert_that(config.get(field), condition) expect("folder.tmp", equal_to("/tmp2")) expect("folder.subdirs.dir2", equal_to("<_<")) expect("folder.subdirs.dir1", equal_to("/o_o")) expect("new_one", equal_to("three")) assert_that(config.get("not_existing", "default"), equal_to("default"))
from pika import credentials as pika_credentials path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.abspath(os.path.join(path, '..'))) from common import config if __name__ == "__main__": parser = argparse.ArgumentParser("Add tasks to queue manually") parser.add_argument("-amqp", "--amqp-server", dest="amqp_server", help="AMQP server address to publish to") parser.add_argument("-s", "--scenario", dest="scenario", help="Scenario to run") parser.add_argument("-c", "--count", dest="count", help="Task count", default=1) parser.add_argument("-amqp-user", "--amqp-user", dest="amqp_user", help="AMQP server user", default="guest") parser.add_argument("-amqp-pass", "--amqp-pass", dest="amqp_pass", help="AMQP server password", default="guest") parser.add_argument("-d", "--drain", dest="drain", action="store_true", help="Drain all previous messages") config.read_config() opts = parser.parse_args() amqp_server = opts.amqp_server if opts.amqp_server else config.get("main.amqp_server") amqp_user = config.get("main.amqp_user", opts.amqp_user) amqp_pass = config.get("main.amqp_pass", opts.amqp_pass) queue = config.get("main.input_queue", "spikeTasks") if not amqp_server: print "FATAL: amqp server is not defined" exit(1) connection = pika.BlockingConnection(pika.ConnectionParameters(host=str(amqp_server), credentials=pika_credentials.PlainCredentials(amqp_user, amqp_pass))) channel = connection.channel()
if os.path.isfile(options.pid_file): logging.error("%s already exists, exiting" % options.pid_file) sys.exit() else: file(options.pid_file, 'w').write(pid) files = [] if options.config: files.append(options.config) if options.log_file: root = logging.getLogger() root.setLevel(logging.DEBUG) formatter = logging.Formatter('[%(asctime)s][%(name)s][%(levelname)s][%(process)d] %(message)s') log_file = handlers.RotatingFileHandler(options.log_file) log_file.setLevel(logging.INFO) log_file.setFormatter(formatter) root.addHandler(log_file) else: logging.basicConfig(level=logging.INFO) config.read_config(*files) try: spike = Spike() spike.run() finally: if options.pid_file and os.path.isfile(options.pid_file): os.unlink(options.pid_file)