def push_series_studylevel(triggered_rules, file_list, series_UID, tags_list): """Prepeares study-level routing for the current series.""" # Move series into individual study-level folder for every rule for current_rule in triggered_rules: if config.mercure[mercure_config.RULES][current_rule].get( mercure_rule.ACTION_TRIGGER, mercure_options.SERIES) == mercure_options.STUDY: folder_name = series_UID + mercure_defs.SEPARATOR + current_rule if (not os.path.exists(folder_name)): try: os.mkdir(folder_name) except: logger.error(f'Unable to create folder {folder_name}') monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create folder {folder_name}') continue try: lock_file = Path(folder_name / mercure_names.LOCK) lock = helper.FileLock(lock_file) except: # Can't create lock file, so something must be seriously wrong logger.error(f'Unable to create lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create lock file {lock_file}') return push_files(file_list, folder_name, (len(triggered_rules) > 1)) lock.free()
def write_configfile(json_content): """Rewrites the config file using the JSON data passed as argument. Used by the config editor of the webgui.""" configuration_file = Path(configuration_filename) # Check for existence of lock file lock_file=Path(configuration_file.parent/configuration_file.stem).with_suffix(mercure_names.LOCK) if lock_file.exists(): raise ResourceWarning(f"Configuration file locked: {lock_file}") try: lock=helper.FileLock(lock_file) except: raise ResourceWarning(f"Unable to lock configuration file: {lock_file}") with open(configuration_file, "w") as json_file: json.dump(json_content, json_file, indent=4) monitor.send_event(monitor.h_events.CONFIG_UPDATE, monitor.severity.INFO, "Wrote configuration file.") logger.info(f"Wrote configuration into: {configuration_file}") try: lock.free() except: # Can't delete lock file, so something must be seriously wrong logger.error(f'Unable to remove lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to remove lock file {lock_file}') return
def route_error_files(): """ Looks for error files, moves these files and the corresponding DICOM files to the error folder, and sends an alert to the bookkeeper instance. """ error_files_found = 0 for entry in os.scandir(config.mercure['incoming_folder']): if entry.name.endswith(".error") and not entry.is_dir(): # Check if a lock file exists. If not, create one. lock_file=Path(config.mercure['incoming_folder'] / entry.name + mercure_names.LOCK) if lock_file.exists(): continue try: lock=helper.FileLock(lock_file) except: continue logger.error(f'Found incoming error file {entry.name}') error_files_found += 1 shutil.move(config.mercure['incoming_folder'] + '/' + entry.name, config.mercure['error_folder'] + '/' + entry.name) dicom_filename = entry.name[:-6] dicom_file = Path(config.mercure['incoming_folder'] + '/' + dicom_filename) if dicom_file.exists(): shutil.move(config.mercure['incoming_folder'] + '/' + dicom_filename, config.mercure['error_folder'] + '/' + dicom_filename) lock.free() if error_files_found > 0: monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Error parsing {error_files_found} incoming files') return
def push_series_discard(fileList,series_UID,discard_series): """Discards the series by moving all files into the "discard" folder, which is periodically cleared.""" # Define the source and target folder. Use UUID as name for the target folder in the # discard directory to avoid collisions discard_path =config.mercure['discard_folder'] + '/' + str(uuid.uuid1()) discard_folder=discard_path + '/' source_folder =config.mercure['incoming_folder'] + '/' # Create subfolder in the discard directory and validate that is has been created try: os.mkdir(discard_path) except Exception: logger.exception(f'Unable to create outgoing folder {discard_path}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create discard folder {discard_path}') return if not Path(discard_path).exists(): logger.error(f'Creating discard folder not possible {discard_path}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Creating discard folder not possible {discard_path}') return # Create lock file in destination folder (to prevent the cleaner module to work on the folder). Note that # the DICOM series in the incoming folder has already been locked in the parent function. try: lock_file=Path(discard_path / mercure_names.LOCK) lock=helper.FileLock(lock_file) except: # Can't create lock file, so something must be seriously wrong logger.error(f'Unable to create lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create lock file in discard folder {lock_file}') return info_text = "" if discard_series: info_text = "Discard by rule " + discard_series monitor.send_series_event(monitor.s_events.DISCARD, series_UID, len(fileList), "", info_text) for entry in fileList: try: shutil.move(source_folder+entry+mercure_names.DCM,discard_folder+entry+mercure_names.DCM) shutil.move(source_folder+entry+mercure_names.TAGS,discard_folder+entry+mercure_names.TAGS) except Exception: logger.exception(f'Problem while discarding file {entry}') logger.exception(f'Source folder {source_folder}') logger.exception(f'Target folder {discard_folder}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Problem during discarding file {entry}') monitor.send_series_event(monitor.s_events.MOVE, series_UID, len(fileList), discard_path, "") try: lock.free() except: # Can't delete lock file, so something must be seriously wrong logger.error(f'Unable to remove lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to remove lock file {lock_file}') return
def process_series(folder): logger.info(f'Now processing = {folder}') lock_file = Path(folder / mercure_names.PROCESSING) if lock_file.exists(): logger.warning(f"Folder already contains lockfile {folder}/" + mercure_names.PROCESSING) return try: lock = helper.FileLock(lock_file) except: # Can't create lock file, so something must be seriously wrong logger.error(f'Unable to create lock file {lock_file}') monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create lock file in processing folder {lock_file}') return processing_success = True needs_dispatching = False # TODO: Perform the processing time.sleep(10) # TODO: Error handling # Create a new lock file to ensure that no other process picks up the folder while copying try: lock_file = lock_file = Path(folder / mercure_names.LOCK_EXTENSION) lock_file.touch() except: logger.info(f"Error locking folder to be moved {folder}") monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Error locking folder to be moved {folder}") # Remove the processing lock lock.free() if not processing_success: move_folder(folder, config.mercure['error_folder']) else: if needs_dispatching: move_folder(folder, config.mercure['outgoing_folder']) else: move_folder(folder, config.mercure['success_folder']) logger.info(f'Done processing case') return
def push_series_studylevel(triggered_rules, file_list, series_UID, tags_list): """Prepeares study-level routing for the current series.""" # Move series into individual study-level folder for every rule for current_rule in triggered_rules: if config.mercure[mercure_config.RULES][current_rule].get( mercure_rule.ACTION_TRIGGER, mercure_options.SERIES) == mercure_options.STUDY: first_series = False # Create folder to buffer the series until study completion study_UID = tags_list["StudyInstanceUID"] folder_name = study_UID + mercure_defs.SEPARATOR + current_rule if not os.path.exists(folder_name): try: os.mkdir(folder_name) first_series = True except: logger.error(f"Unable to create folder {folder_name}") monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Unable to create folder {folder_name}") continue lock_file = Path(folder_name) / mercure_names.LOCK try: lock = helper.FileLock(lock_file) except: # Can't create lock file, so something must be seriously wrong logger.error(f"Unable to create lock file {lock_file}") monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Unable to create lock file {lock_file}") continue if first_series: # Create task file with information on complete criteria create_study_task(folder_name, current_rule, study_UID, tags_list) else: # Add data from latest series to task file update_study_task(folder_name, current_rule, study_UID, tags_list) # Copy (or move) the files into the study folder push_files(file_list, folder_name, (len(triggered_rules) > 1)) lock.free()
def save_config(): """Saves the current configuration in a file on the disk. Raises an exception if the file has been locked by another process.""" global configuration_timestamp configuration_file = Path(configuration_filename) # Check for existence of lock file lock_file = Path(configuration_file.parent / configuration_file.stem).with_suffix(mercure_names.LOCK) if lock_file.exists(): raise ResourceWarning(f"Configuration file locked: {lock_file}") try: lock = helper.FileLock(lock_file) except: raise ResourceWarning( f"Unable to lock configuration file: {lock_file}") with open(configuration_file, "w") as json_file: json.dump(mercure, json_file, indent=4) try: stat = os.stat(configuration_file) configuration_timestamp = stat.st_mtime except AttributeError: configuration_timestamp = 0 monitor.send_event(monitor.h_events.CONFIG_UPDATE, monitor.severity.INFO, "Saved new configuration.") logger.info(f"Stored configuration into: {configuration_file}") try: lock.free() except: # Can't delete lock file, so something must be seriously wrong logger.error(f'Unable to remove lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to remove lock file {lock_file}') return
def push_serieslevel_outgoing(triggered_rules, file_list, series_UID, tags_list, selected_targets): """Move the DICOM files of the series to a separate subfolder for each target in the outgoing folder.""" source_folder = config.mercure['incoming_folder'] + '/' # Determine if the files should be copied or moved. If only one rule triggered, files can # safely be moved, otherwise files will be moved and removed in the end move_operation = False if len(triggered_rules) == 1: move_operation = True for target in selected_targets: if not target in config.mercure["targets"]: logger.error(f"Invalid target selected {target}") monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Invalid target selected {target}") continue folder_name = config.mercure['outgoing_folder'] + '/' + str( uuid.uuid1()) target_folder = folder_name + "/" try: os.mkdir(folder_name) except Exception: logger.exception(f'Unable to create outgoing folder {folder_name}') monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create outgoing folder {folder_name}') return if not Path(folder_name).exists(): logger.error(f'Creating folder not possible {folder_name}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Creating folder not possible {folder_name}') return try: lock_file = Path(folder_name / mercure_names.LOCK) lock = helper.FileLock(lock_file) except: # Can't create lock file, so something must be seriously wrong logger.error(f'Unable to create lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create lock file {lock_file}') return # Generate task file with dispatch information task_filename = target_folder + mercure_names.TASKFILE task_json = generate_taskfile_route(series_UID, mercure_options.SERIES, selected_targets[target], tags_list, target) try: with open(task_filename, 'w') as task_file: json.dump(task_json, task_file) except: logger.error(f"Unable to create task file {task_filename}") monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Unable to create task file {task_filename}") continue monitor.send_series_event(monitor.s_events.ROUTE, series_UID, len(file_list), target, selected_targets[target]) if move_operation: operation = shutil.move else: operation = shutil.copy for entry in file_list: try: operation(source_folder + entry + mercure_names.DCM, target_folder + entry + mercure_names.DCM) operation(source_folder + entry + mercure_names.TAGS, target_folder + entry + mercure_names.TAGS) except Exception: logger.exception( f'Problem while pushing file to outgoing {entry}') logger.exception(f'Source folder {source_folder}') logger.exception(f'Target folder {target_folder}') monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Problem while pushing file to outgoing {entry}') monitor.send_series_event(monitor.s_events.MOVE, series_UID, len(file_list), folder_name, "") try: lock.free() except: # Can't delete lock file, so something must be seriously wrong logger.error(f'Unable to remove lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to remove lock file {lock_file}') return
def push_serieslevel_processing(triggered_rules, file_list, series_UID, tags_list): for current_rule in triggered_rules: if config.mercure[mercure_config.RULES][current_rule].get( mercure_rule.ACTION_TRIGGER, mercure_options.SERIES) == mercure_options.SERIES: if ((config.mercure[mercure_config.RULES][current_rule].get( mercure_rule.ACTION, "") == mercure_actions.PROCESS) or (config.mercure[mercure_config.RULES][current_rule].get( mercure_rule.ACTION, "") == mercure_actions.BOTH)): # Determine if the files should be copied or moved. If only one rule triggered, files can # safely be moved, otherwise files will be moved and removed in the end copy_files = True if len(triggered_rules) == 1: copy_files = False folder_name = config.mercure[ mercure_folders.PROCESSING] + '/' + str(uuid.uuid1()) target_folder = folder_name + "/" try: os.mkdir(folder_name) except Exception: logger.exception( f'Unable to create outgoing folder {folder_name}') monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create processing folder {folder_name}') return if not Path(folder_name).exists(): logger.error(f'Creating folder not possible {folder_name}') monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Creating folder not possible {folder_name}') return try: lock_file = Path(folder_name / mercure_names.LOCK) lock = helper.FileLock(lock_file) except: # Can't create lock file, so something must be seriously wrong logger.error(f'Unable to create lock file {lock_file}') monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create lock file {lock_file}') return # Generate task file with dispatch information task_filename = target_folder + mercure_names.TASKFILE task_json = generate_taskfile_process(series_UID, mercure_options.SERIES, current_rule, tags_list) try: with open(task_filename, 'w') as task_file: json.dump(task_json, task_file) except: logger.error(f"Unable to create task file {task_filename}") monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Unable to create task file {task_filename}") continue if (not push_files(file_list, target_folder, copy_files)): logger.error( f'Unable to push files into processing folder {target_folder}' ) monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to push files into processing folder {target_folder}' ) return lock.free() trigger_serieslevel_notification_reception( current_rule, tags_list)
def route_series(series_UID): """Processes the series with the given series UID from the incoming folder.""" lock_file = Path(config.mercure['incoming_folder'] + '/' + str(series_UID) + mercure_names.LOCK) if lock_file.exists(): # Series is locked, so another instance might be working on it return try: lock = helper.FileLock(lock_file) except: # Can't create lock file, so something must be seriously wrong logger.error(f'Unable to create lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to create lock file {lock_file}') return logger.info(f'Processing series {series_UID}') fileList = [] seriesPrefix = series_UID + "#" # Collect all files belonging to the series for entry in os.scandir(config.mercure['incoming_folder']): if entry.name.endswith(mercure_names.TAGS) and entry.name.startswith( seriesPrefix) and not entry.is_dir(): stemName = entry.name[:-5] fileList.append(stemName) logger.info("DICOM files found: " + str(len(fileList))) # Use the tags file from the first slice for evaluating the routing rules tagsMasterFile = Path(config.mercure['incoming_folder'] + '/' + fileList[0] + mercure_names.TAGS) if not tagsMasterFile.exists(): logger.error(f'Missing file! {tagsMasterFile.name}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Missing file {tagsMasterFile.name}') return try: with open(tagsMasterFile, "r") as json_file: tagsList = json.load(json_file) except Exception: logger.exception(f"Invalid tag information of series {series_UID}") monitor.send_series_event(monitor.s_events.ERROR, entry, 0, "", "Invalid tag information") monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Invalid tag for series {series_UID}") return monitor.send_register_series(tagsList) monitor.send_series_event(monitor.s_events.REGISTERED, series_UID, len(fileList), "", "") discard_series = "" # Now test the routing rules and evaluate which rules have been triggered. If one of the triggered # rules enforces discarding, discard_series will be True. triggered_rules, discard_series = get_triggered_rules(tagsList) if (len(triggered_rules) == 0) or (discard_series): # If no routing rule has triggered or discarding has been enforced, discard the series push_series_discard(fileList, series_UID, discard_series) else: # Strategy: If only one triggered rule, move files. If multiple, copy files push_series_studylevel(triggered_rules, fileList, series_UID, tagsList) push_series_serieslevel(triggered_rules, fileList, series_UID, tagsList) if (len(triggered_rules) > 1): remove_series(fileList) try: lock.free() except: # Can't delete lock file, so something must be seriously wrong logger.error(f'Unable to remove lock file {lock_file}') monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f'Unable to remove lock file {lock_file}') return
def process_series(folder): logger.info(f"Now processing = {folder}") docker_client = docker.from_env() lock_file = Path(folder) / mercure_names.PROCESSING if lock_file.exists(): logger.warning(f"Folder already contains lockfile {folder}/" + mercure_names.PROCESSING) return try: lock = helper.FileLock(lock_file) except: # Can't create lock file, so something must be seriously wrong logger.error(f"Unable to create lock file {lock_file}") logger.error(traceback.format_exc()) monitor.send_event( monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Unable to create lock file in processing folder {lock_file}") return processing_success = True needs_dispatching = False # TODO: Perform the processing time.sleep(10) def get_task(): the_path = Path(folder) / mercure_names.TASKFILE if not the_path.exists(): return None with open(the_path, "r") as f: return json.load(f) task = get_task() logger.info(task["process"]["docker_tag"]) docker_client.containers.run( task["process"]["docker_tag"], "--dicom-path /data", volumes={folder: { "bind": "/data", "mode": "rw" }}) # TODO: Error handling # Create a new lock file to ensure that no other process picks up the folder while copying lock_file = Path(folder) / mercure_names.LOCK try: lock_file.touch() except: logger.info(f"Error locking folder to be moved {folder}") logger.error(traceback.format_exc()) monitor.send_event(monitor.h_events.PROCESSING, monitor.severity.ERROR, f"Error locking folder to be moved {folder}") # Remove the processing lock lock.free() if not processing_success: move_folder(folder, config.mercure["error_folder"]) else: if needs_dispatching: move_folder(folder, config.mercure["outgoing_folder"]) else: move_folder(folder, config.mercure["success_folder"]) logger.info(f"Done processing case") return