def generate_docker_cmd(algorithm_info, to_detect): cmd = lookup.get_cmd(algorithm_info) new_cmd = generator.replace(cmd, to_detect) if lookup.PIPE_OUTPUT in algorithm_info: #need to return a native command for now -- HACKY FIX!! result_file_path = generator.generate_result_file( algorithm_info, to_detect, 'txt') write_to_result_cmd = ' > ' + result_file_path docker_cmd = [lookup.docker_exec_prefix] if lookup.WORKING_DIR in algorithm_info: docker_cmd.append('-w ' + algorithm_info[lookup.WORKING_DIR]) docker_cmd += [ str(to_detect[lookup.container_id]), new_cmd, write_to_result_cmd ] docker_cmd = ' '.join(docker_cmd) return { lookup.COMMAND_TYPE: lookup.NATIVE, lookup.COMMAND: [docker_cmd] } params = [to_detect[lookup.container_id], new_cmd] if lookup.WORKING_DIR in algorithm_info: params.append(algorithm_info[lookup.WORKING_DIR]) return {lookup.COMMAND_TYPE: lookup.DOCKER, lookup.COMMAND: params}
def generate_result_file(algorithm_info, to_detect, file_type, temp=False): output_file_name = algorithm_info[lookup.uuid_descriptor] cmd = lookup.get_cmd(algorithm_info) if algorithm_info[ lookup. COMMAND_TYPE] == lookup.DOCKER and lookup.PIPE_OUTPUT not in algorithm_info: #hacky fix for piping output output_directory = lookup.result_dir else: output_directory = abspath( lookup.get_algo_asset_dirs()[lookup.detector]) if lookup.INPUT_IMAGE_PATH in cmd: output_file_name += '_' + fs.get_filename( to_detect[lookup.INPUT_IMAGE_PATH], extension=False) elif lookup.INPUT_IMAGE_DIRECTORY in cmd: output_file_name += '_' + fs.get_filename( to_detect[lookup.INPUT_IMAGE_DIRECTORY]) if temp: output_file_name += '-temp' output_file_name += '.' + file_type return join(output_directory, output_file_name)
def preprocess_native(algorithm_info, to_detect_list): #probably need to transform from directory to list, vice versa. pre_cmd = lookup.get_pre_cmd(algorithm_info) cmd = lookup.get_cmd(algorithm_info) updated_detect_list = to_detect_list if pre_cmd: update_write_file(algorithm_info, pre_cmd, updated_detect_list) if lookup.INPUT_IMAGE_DIRECTORY in cmd: updated_detect_list = generator.get_directories(to_detect_list) update_write_file(algorithm_info, cmd, updated_detect_list) return [], updated_detect_list
def compile_results(algorithm_info, source_db): cmd = lookup.get_cmd(algorithm_info) post_cmd = lookup.get_post_cmd(algorithm_info) if not post_cmd: post_cmd = '' if lookup.INPUT_IMAGE_PATH in cmd: if lookup.PIPE_OUTPUT in algorithm_info or lookup.RESULT_TXT_FILE in cmd or lookup.RESULT_TXT_FILE in post_cmd: return compile_txt_results(algorithm_info, source_db) elif lookup.RESULT_CSV_FILE in cmd or lookup.RESULT_CSV_FILE in post_cmd: return compile_csv_results(algorithm_info, source_db) elif lookup.INPUT_IMAGE_DIRECTORY in cmd: return compile_csv_directory(algorithm_info, source_db) raise NotImplementedError
def termination_native(algorithm_info, detected_list): termination_cmds = [] if lookup.PIPE_OUTPUT in algorithm_info: cmd = lookup.get_cmd(algorithm_info) for detected in detected_list: result_file = generator.generate_result_file( algorithm_info, detected, 'txt') removal_cmd = ' '.join([lookup.removal_prefix, result_file]) termination_cmds.append({ lookup.COMMAND_TYPE: lookup.NATIVE, lookup.COMMAND: [removal_cmd] }) for detected in detected_list: if lookup.TEMP_CSV_FILE in detected: removal_cmd = ' '.join( [lookup.removal_prefix, detected[lookup.TEMP_CSV_FILE]]) termination_cmds.append({ lookup.COMMAND_TYPE: lookup.NATIVE, lookup.COMMAND: [removal_cmd] }) if lookup.TEMP_TXT_FILE in detected: removal_cmd = ' '.join( [lookup.removal_prefix, detected[lookup.TEMP_TXT_FILE]]) termination_cmds.append({ lookup.COMMAND_TYPE: lookup.NATIVE, lookup.COMMAND: [removal_cmd] }) if lookup.RESULT_CSV_FILE in detected: removal_cmd = ' '.join( [lookup.removal_prefix, detected[lookup.RESULT_CSV_FILE]]) termination_cmds.append({ lookup.COMMAND_TYPE: lookup.NATIVE, lookup.COMMAND: [removal_cmd] }) if lookup.RESULT_TXT_FILE in detected: removal_cmd = ' '.join( [lookup.removal_prefix, detected[lookup.RESULT_TXT_FILE]]) termination_cmds.append({ lookup.COMMAND_TYPE: lookup.NATIVE, lookup.COMMAND: [removal_cmd] }) return termination_cmds
def generate_native_cmd(algorithm_info, to_detect): #need to do regular substitutions cmd = lookup.get_cmd(algorithm_info) new_cmd = generator.replace(cmd, to_detect) if lookup.PIPE_OUTPUT in algorithm_info: result_file = generator.generate_result_file(algorithm_info, to_detect, 'txt') write_to_result_cmd = ' > ' + result_file new_cmd += write_to_result_cmd if lookup.WORKING_DIR in algorithm_info: wdir = algorithm_info[lookup.WORKING_DIR] new_cmd = ['(', 'cd', wdir, '&&', new_cmd, ')'] new_cmd = ' '.join(new_cmd) return {lookup.COMMAND_TYPE: lookup.NATIVE, lookup.COMMAND: [new_cmd]}
def preprocess_docker(algorithm_info, to_detect_list): """starts docker command and updates parameters appropriately""" image_name = algorithm_info[lookup.DOCKER_IMAGE] cmd = lookup.get_cmd(algorithm_info) volumes = {} if lookup.INPUT_IMAGE_DIRECTORY in cmd: updated_detect_list = generator.get_directories(to_detect_list) for updated_detect in updated_detect_list: docker_directory = '/' + fs.get_uuid() volumes[updated_detect[lookup.INPUT_IMAGE_DIRECTORY]] = { 'bind': docker_directory, 'mode': 'rw' } updated_detect[lookup.INPUT_IMAGE_DIRECTORY] = docker_directory elif lookup.INPUT_IMAGE_PATH in cmd: for to_detect in to_detect_list: original_input_path = to_detect[lookup.INPUT_IMAGE_PATH] original_input_path = abspath(original_input_path) local_input_dir = fs.get_directory(original_input_path) volumes[local_input_dir] = {'bind': lookup.input_dir, 'mode': 'rw'} input_filename = fs.get_filename(original_input_path) new_input_path = join(lookup.input_dir, input_filename) to_detect[lookup.INPUT_IMAGE_PATH] = new_input_path result_directory = abspath(lookup.get_algo_asset_dirs()[lookup.detector]) assert (fs.dir_exists(result_directory)) volumes[result_directory] = {'bind': lookup.result_dir, 'mode': 'rw'} container_id = runner.start_docker(image_name, volumes=volumes) for to_detect in to_detect_list: to_detect[lookup.container_id] = container_id return [], to_detect_list