def ask_for_img_path(self): """ Ask user to pick directory, check its validity and load its containing images. :return: """ while True: img_path = str( QFileDialog.getExistingDirectory(self, "Select image directory", STD_IMG_DIRECTORY)) if not img_path: # stop program, if user clicks 'abort' and self.img_path is not initialized sys.exit() if not img_path: # ignore, if user clicks 'abort' break if len(utils.list_directory(img_path, [ '.jpg' ])) > 0: # if chosen directory contains JPEG files, break loop self.img_directory = img_path self.img_num = len(utils.list_directory(img_path, ['.jpg'])) self.reset_img_index() self.set_img() break else: logging.info( "The selected directory doesn\'t contain JPEG files. Please choose another one." )
def manual_process(cbf_path): # potential additions: # check to see if stills_process input = experiments.json works cbf_path = '/dls/i24/data/2017/nt14493-63/CuNIR/waylinCD/' reference_path = '/dls/i24/data/2017/nt14493-63/processing/stills_process/waylinCD/' wd = '/dls/i24/data/2017/nt14493-63/processing/scripts/experiment_refinement_process_test_folder/waylinCD/' experiments_json_list = nest.find_experiments_json_starts('waylinCD', doses=20, before_limit=5) all_out_files = utils.list_directory(wd) for file_num in experiments_json_list: diff = file_num % 20 start_num = file_num - diff final_num = start_num + 20 file_num = '%.5d' % file_num file_list = [ 'waylinCD0077_%.5d' % num for num in range(start_num, final_num) ] ref_exp_json = '%sidx-waylinCD0077_%s_refined_experiments.json' % ( reference_path, file_num ) # idx-waylinCD0077_46664_refined_experiments.json print ref_exp_json for file in file_list: num = file.split('_')[1] idx_file = utils.idx_file_check(all_out_files, num) if idx_file == []: p_file_name = sg.write_manual_process_file( wd, cbf_path, file, ref_exp_json) # run manual process utils.run_cluster_job_with_log(p_file_name) utils.check_log(job_lim=20) # default job limit = 50 # move_cluster_logs() if file == file_list[-1]: break
def main(): """ This method creates the YOLO file 'test.txt' containing all file names inside the given directory. YOLO needs this file to know all images, which are necessary to test detection performance. :return: """ # setup logging utils.setup_logging() # setup file dialog dirname = os.path.dirname(__file__) initial_dir = os.path.join(dirname, os.path.realpath('../')) # select path to object images test_images_path = filedialog.askdirectory(initialdir=initial_dir, title="Select image directory") test_img_path = os.path.join(test_images_path).replace('\\', '/') # select path to output directory output_path = os.path.join(test_images_path) # collect all file names inside given directory content_str = '' filename_list = utils.list_directory(test_img_path, [".jpg"]) # merge file names with darknet folder structure for curr_filename in filename_list[:-1]: content_str += "%s\n" % os.path.join(DARKNET_IMG_DIRECTORY, curr_filename) content_str += "%s" % os.path.join(DARKNET_IMG_DIRECTORY, filename_list[-1]) utils.create_txt_file(output_path, OUTPUT_FILENAME, content_str) logging.info("Successfully created \'test.txt\' in \'%s\'" % output_path)
def _run(self): records = [] for file_name in list_directory(self.path): for record in tf.python_io.tf_record_iterator(file_name): records.append(record) if len(records) == self.batch_size: yield records records = [] if len(records) > 0: yield records
def _run(self, path): dataset = tf.data.TFRecordDataset(list_directory(path)) if self.skip_size: dataset = dataset.skip(self.skip_size) dataset = dataset.map(self.components['record_to_tensor'].run, num_parallel_calls=self.n_processes) if self.shuffle_flag: dataset = dataset.shuffle(self.buffer_size) if self.repeat_flag: dataset = dataset.repeat() dataset = dataset.batch(self.batch_size).prefetch(self.n_processes) return dataset.make_one_shot_iterator().get_next()
def apply_overlay_filter(overlay_filter, img): """ Apply overlay filter with given overlay_filter parameters. :param overlay_filter: object containing overlay parameters :param img: passed image on which overlay filter should be applied :return: img: processed image overlayed with another image """ # get overlay image if overlay_filter.path_type == 0: # random image out of overlay_img_path overlay_img_path_list = utils.list_directory( overlay_filter.overlay_img_path, file_ext_list=['.jpg', '.png']) random_overlay_idx = random.randint(0, len(overlay_img_path_list) - 1) overlay_img_filename = overlay_img_path_list[random_overlay_idx] overlay_img_path = os.path.abspath(overlay_filter.overlay_img_path + overlay_img_filename) elif overlay_filter.path_type == 1: # overlay image path specified in overlay_filter.overlay_img_path overlay_img_path = overlay_filter.overlay_img_path else: logging.error("ERROR: Unknown path type (%s) of overlay filter" % overlay_filter.path_type) sys.exit(-1) # prepare intensity intensity = utils.to_float(overlay_filter.intensity / 100) # read in overlapping image overlay_img = img_processing.import_image(overlay_img_path) overlay_img = img_processing.resize_aspect_ratio_crop_image( overlay_img, img.shape[1], img.shape[0]) # filter if overlay_filter.filter == 0: # multiply img = img_processing.multiply_images(img, overlay_img, intensity) elif overlay_filter.filter == 1: # color img = img_processing.color_filter(img, overlay_img, intensity) elif overlay_filter.filter == 2: # brighten img = img_processing.brighten_filter(img, overlay_img, intensity) elif overlay_filter.filter == 3: # darken img = img_processing.darken_filter(img, overlay_img, intensity) else: logging.error("ERROR: Unknown filter (%s) of overlay filter" % overlay_filter.filter) sys.exit(-1) target_list = ['fore- and background', 'foreground', 'background'] filter_list = ['multiply', 'color', 'brighten', 'darken'] logging.info( " => apply overlay filter \'%s\' on %s with overlay image \'%s\'" % (filter_list[overlay_filter.filter], target_list[overlay_filter.target], overlay_img_path)) return img
def set_img(self): """ Load current image by image index, resize it and create checkerboard image with the same shape. :return: """ img_path_list = utils.list_directory(self.img_directory, ['.jpg']) img_path = img_path_list[self.img_index] img = cv2.imread(os.path.join(self.img_directory, img_path), cv2.IMREAD_UNCHANGED) self.img = img_processing.resize_image_aspect_ratio( img, new_width=WINDOW_WIDTH) self.img_trans = img_processing.create_checkerboard_pattern( self.img.shape) self.update_img()
def apply_add_obj_filter(add_obj_filter, bg_img, dest_res_x, dest_res_y): """ Apply 'Add object' filter with given add_obj_filter parameters, a given background image and the desired output resolution. The add_obj object can contain almost all other filters, which get applied on each added object. :param add_obj_filter: object containing add_obj parameters :param bg_img: background image :param dest_res_x: x resolution of output image :param dest_res_y: y resolution of output image :return: bg_img: background image containing all inserted objects """ # get number of wanted additional objects if add_obj_filter.obj_min == add_obj_filter.obj_max: num_obj = int(add_obj_filter.obj_min) else: num_obj = random.randint(add_obj_filter.obj_min, add_obj_filter.obj_max) logging.info(" => add %d additional objects" % num_obj) # create list of objects containing 'add_obj' object images obj_list = [] # add objects to the background for obj_counter in range(num_obj): # get object image if add_obj_filter.path_type == 0: add_obj_path_list = utils.list_directory( add_obj_filter.add_obj_path, file_ext_list=['.jpg', '.png']) random_add_obj_idx = random.randint(0, len(add_obj_path_list) - 1) add_obj_img_path = os.path.abspath( add_obj_filter.add_obj_path + add_obj_path_list[random_add_obj_idx]) else: add_obj_img_path = add_obj_filter.add_obj_path add_obj_img = img_processing.import_image(add_obj_img_path) obj = Object(filename='', image=add_obj_img, bbox=None, category=None) obj_list.append(obj) # add overlap and add_obj filter so method 'apply_filters_on_objects()' can handle this add_obj_filter.overlap = None add_obj_filter.add_obj = None # add chosen object images to background image merged_img, obj_list = apply_filters_on_objects(obj_list, add_obj_filter, bg_img, dest_res_x, dest_res_y) return merged_img
def main(): """ Resize all images inside the selected directory to the desired new width retaining original aspect ratio. :return: """ # setup logging utils.setup_logging() # select path to object images dirname = os.path.dirname(__file__) initial_dir = os.path.join(dirname, os.path.realpath('../images/')) obj_img_path = filedialog.askdirectory( initialdir=initial_dir, title="Select object image directory") # ask desired resolution img_width_str = simpledialog.askstring("Resolution input", "Enter desired image width") if not img_width_str: sys.exit() img_width = utils.to_int(img_width_str) # create folder for resized images foldername = "resized_width_%s" % img_width_str folder_path = os.path.join(obj_img_path, foldername) create_folder(folder_path) # get object images obj_img_filename_list = utils.list_directory(obj_img_path, ['.jpg', '.png']) # resize object images counter = 1 total = len(obj_img_filename_list) for obj_img_filename in obj_img_filename_list: obj_img = img_processing.import_image( os.path.join(obj_img_path, obj_img_filename)) obj_img_resized = img_processing.resize_image_aspect_ratio( obj_img, new_width=img_width) img_processing.save_image(obj_img_resized, folder_path, obj_img_filename) logging.info("=> %s%% done" % str(int((counter / total) * 100))) counter += 1 logging.info( "\n\nSuccessfully resized all images inside \'%s\' to width %s" % (obj_img_path, img_width))
def generate_train_data_file(self, output_path): """ Generate 'train.txt' file containing paths to all images, on which YOLO gets trained later. :param output_path: path, where YOLO output files should be stored :return: """ logging.info("Generating YOLO text file \'%s\' to \'%s\'" % (self.TRAIN_TXT_FILENAME, output_path)) content_str = '' filename_list = utils.list_directory(output_path, [".jpg"]) for curr_filename in filename_list[:-1]: content_str += "%s%s\n" % (self.TRAIN_DATA_PATH, curr_filename) content_str += "%s%s" % (self.TRAIN_DATA_PATH, filename_list[-1] ) # last element without newline filename = self.TRAIN_TXT_FILENAME.split('.')[0] utils.create_txt_file(output_path, filename, content_str)
def set_img_directory(self): """ Ask user to pick valid image folder and then load images in chosen directory. :return: """ path_valid_bool = False while not path_valid_bool: # if img_path is invalid, let user choose another one directory_file_num = len( utils.list_directory(self.img_directory, ['.jpg'], error_quit=False)) if directory_file_num > 0: self.img_num = directory_file_num self.reset_img_index() self.set_img() path_valid_bool = True else: self.ask_for_img_path()
def auto_process(visit_directory, protein_name, chip_name, chip_run, output_directory, job_lim=20, iteration_lim=1000, wait_time=10, log_name='03102017_out.txt' ): #needs checking, auto_process need rewriting cbf_path = os.path.join(visit_directory, protein_name, chip_name) cbf_list = utils.list_directory_files_of_type(cbf_path, '.cbf') wd = os.path.join(visit_directory, 'processing/stills_process/', output_directory) all_out_files = utils.list_directory(wd) list_to_append = [] print cbf_path, wd for file in cbf_list: idx_file = utils.idx_file_check(all_out_files, file.split('.')[0]) if idx_file == []: list_to_append.append(file) job_length = len(list_to_append) if job_length >= 1 or file == cbf_path[-1]: print file initial_num = list_to_append[0].split('.')[0].split('_')[1] final_num = list_to_append[-1].split('.')[0].split('_')[1] if chip_run is None: chip_run = cbf_list[0].split('_')[0] #chipname+run number else: chip_run = "%.4d" % chip_run p_file_name = sg.write_multi_stills_file( wd, cbf_path, chip_run, initial_num, final_num) list_to_append = [] utils.check_log(log_file_name='03102017_out.txt', job_lim=20, i=0, iteration_lim=1000, wait_time=10) utils.run_cluster_job_with_log(p_file_name)
def recent_model_dir(dir): folder_list = utils.list_directory(dir, True) folder_list = sorted(folder_list, key=get_epoch) return folder_list[-1]
def list_dir(): res = utils.list_directory(request.args.get('folder', settings.folder)) return jsonify(**res)
def generate(general_struct, filter_chain_list, class_id_to_name_dict, img_name_to_class_id_dict, yolo_version, draw_bounding_boxes=False, no_yolo_output=False): """ Generate images with given settings in general struct and filter chain list. With the assignment of each object image to an object class this method also creates output files for YOLO. :param general_struct: object containing general settings :param filter_chain_list: list of filter_chain objects containing all settings for every set filter :param class_id_to_name_dict: dict assigning object class IDs to its corresponding class name :param img_name_to_class_id_dict: dict assigning each image to its object class ID :param yolo_version: number of YOLO version for which the YOLO output files should be created :param draw_bounding_boxes: boolean value specifying whether bounding boxes should be drawn around object :param no_yolo_output: boolean value specifying whether no YOLO output files should be created :return: """ # get destination resolution dest_res_x = general_struct.output_width dest_res_y = general_struct.output_height # collect input images objects_path = general_struct.object_path obj_img_filename_list = utils.list_directory(objects_path, [".jpg", ".png"]) # setup yolo object yolo_generator = YoloGenerator(yolo_version=yolo_version) # set counter for filename of stored images img_counter = 1 # list of all objects obj_list = [] # do operations for every object for filter_chain in filter_chain_list: for obj_img_filename in obj_img_filename_list: # TODO: following outprint should be in the next upper loop and here should be "Generate image x" or smth logging.info("Process filter chain number %d:" % img_counter) output_filename = str(img_counter) obj_list.clear() # list of all objects # get number of objects which should be placed onto each generated image num_obj_per_img_min, num_obj_per_img_max = utils.extract_range( filter_chain.num_obj_per_img) if num_obj_per_img_min == num_obj_per_img_max: num_obj_per_img = int(num_obj_per_img_min) else: num_obj_per_img = random.randint(num_obj_per_img_min, num_obj_per_img_max) logging.info( " => %s objects get placed onto each generated image" % num_obj_per_img) ############################################################# # BACKGROUND FILTERS # get resolution out of resolution filter if filter_chain.resolution: res_x, res_y = apply_resolution_filter(filter_chain.resolution) else: res_x = dest_res_x res_y = dest_res_y # apply all filters aiming on background bg_img = apply_filters_on_background( general_struct.background_path, filter_chain, res_x, res_y) ############################################################# # OBJECTS FILTERS # read in main object image obj_img_path = os.path.abspath( os.path.join(general_struct.object_path, obj_img_filename)) category_number = img_name_to_class_id_dict[obj_img_filename] main_obj = Object(filename=obj_img_filename, image=img_processing.import_image(obj_img_path), category=category_number) obj_list.append(main_obj) logging.info(" => object image path \'%s\'" % obj_img_filename) # read in other randomly selected object images including their labels for x in range(num_obj_per_img - 1): random_idx = random.randint(0, len(obj_img_filename_list) - 1) obj_img_path = os.path.abspath( os.path.join(general_struct.object_path, obj_img_filename_list[random_idx])) category_number = img_name_to_class_id_dict[ obj_img_filename_list[random_idx]] obj = Object(filename=obj_img_filename_list[random_idx], image=img_processing.import_image(obj_img_path), category=category_number) logging.info(" => random object image path \'%s\'" % obj_img_path) obj_list.append(obj) # apply all object filters on the generated objects merged_img, obj_list = apply_filters_on_objects( obj_list, filter_chain, bg_img, res_x, res_y) ############################################################# # OBJECT AND BACKGROUND FILTERS # before applying all other filters, insert additional objects in front of objects for obj in obj_list: # overlap filter applied onto fore- and background if filter_chain.overlap: if filter_chain.overlap.type == 1: # type 1: in front of objects merged_img = apply_overlap_filter( filter_chain.overlap, obj, merged_img) elif filter_chain.overlap.type != 0: logging.error( "ERROR: Unknown overlap filter type (%s)" % filter_chain.overlap.type) sys.exit(-1) # apply all filters aiming for object and background merged_img = apply_filters_on_objects_and_background( merged_img, filter_chain) ############################################################# # draw bounding box if draw_bounding_boxes: for obj in obj_list: img_processing.draw_bounding_box(merged_img, obj.bbox) # save result image img_processing.save_image(merged_img, general_struct.output_path, output_filename + '.jpg') # create YOLO information for every generated image with category numbers and bounding boxes of objects yolo_generator.generate_yolo_output(obj_list, merged_img.shape[:2], general_struct.output_path, output_filename) logging.info("... done processing filter chain number %s\n" % img_counter) img_counter += 1 # create text files 'train.txt', 'obj.names' and 'obj.data' for YOLO inside the output directory if not no_yolo_output: yolo_generator.generate_train_data_file(general_struct.output_path) yolo_generator.generate_obj_names_file(general_struct.output_path, class_id_to_name_dict) yolo_generator.generate_obj_data_file( general_struct.output_path, len(class_id_to_name_dict.keys())) yolo_generator.generate_yolo_obj_cfg(general_struct.output_path, len(class_id_to_name_dict.keys())) return
def apply_overlap_filter(overlap_filter_original, obj, bg_img): """ Apply overlap filter with given overlap_filter parameters. :param overlap_filter: object containing overlap parameters :param obj_img: image containing cropped object :param bg_img: passed image on which overlap filter should be applied :param x_offset_factor: x translation value :param y_offset_factor: y translation value :return: img: processed image with overlapping object """ # only operate on copy of overlap_filter, so the original gets untouched overlap_filter = copy.deepcopy(overlap_filter_original) # extract important information out of object obj_img = obj.image x_offset_factor = obj.x_offset y_offset_factor = obj.y_offset # get overlap image if overlap_filter.path_type == 0: # random image out of overlap_img_path overlap_img_path_list = utils.list_directory( overlap_filter.overlap_img_path, file_ext_list=['.jpg', '.png']) random_overlap_idx = random.randint(0, len(overlap_img_path_list) - 1) overlap_img_filename = overlap_img_path_list[random_overlap_idx] overlap_img_path = os.path.abspath(overlap_filter.overlap_img_path + overlap_img_filename) elif overlap_filter.path_type == 1: # overlap image path specified in overlap_filter.overlap_img_path overlap_img_path = overlap_filter.overlap_img_path else: logging.error("ERROR: Unknown path type (%s) of overlap filter" % overlap_filter.path_type) sys.exit(-1) # get overlapping percentage values # x: if overlap_filter.x_pct_min == overlap_filter.x_pct_max: x_pct = overlap_filter.x_pct_min else: x_pct = random.randint(overlap_filter.x_pct_min, overlap_filter.x_pct_max) # y: if overlap_filter.y_pct_min == overlap_filter.y_pct_max: y_pct = overlap_filter.y_pct_min else: y_pct = random.randint(overlap_filter.y_pct_min, overlap_filter.y_pct_max) y_pct *= -1 # invert y_pct, so positive values overlap upper part and negative values overlap lower part # read in overlapping image overlap_img = img_processing.import_image(overlap_img_path) # calculate scale in relation to object image if obj_img.shape[0] > obj_img.shape[ 1]: # get the larger shape length for scale calculation, so overlap object is big enough if overlap_img.shape[0] > overlap_img.shape[1]: scale_factor = obj_img.shape[0] / overlap_img.shape[0] else: scale_factor = obj_img.shape[0] / overlap_img.shape[1] else: if overlap_img.shape[0] > overlap_img.shape[1]: scale_factor = obj_img.shape[1] / overlap_img.shape[0] else: scale_factor = obj_img.shape[1] / overlap_img.shape[1] # calculate translation to attain overlapping percentage x_len_pct = obj_img.shape[1] * (float(x_pct) / 100) y_len_pct = obj_img.shape[0] * (float(y_pct) / 100) x_rel_len = x_len_pct / (bg_img.shape[1] - overlap_img.shape[1]) y_rel_len = y_len_pct / (bg_img.shape[0] - overlap_img.shape[1]) x_offset_factor = min(max(0, x_offset_factor + x_rel_len), 1) # minimal value must be 0 and maximum must be 1 y_offset_factor = min(max(0, y_offset_factor + y_rel_len), 1) # minimal value must be 0 and maximum must be 1 # update overlap filter with calculated standard values # add translate filter x_range_str = '%s:%s' % (x_offset_factor * 100, x_offset_factor * 100) y_range_str = '%s:%s' % (y_offset_factor * 100, y_offset_factor * 100) overlap_filter.translate = TranslateFilter(x_range_str=x_range_str, y_range_str=y_range_str) # update scale filter with default size if overlap_filter.scale: overlap_filter.scale.min = overlap_filter.scale.min * scale_factor overlap_filter.scale.max = overlap_filter.scale.max * scale_factor else: scale_range_str = '%s:%s' % (scale_factor * 100, scale_factor * 100) overlap_filter.scale = ScaleFilter(scale_range_str=scale_range_str) # add clip and overlap filter so method 'apply_filters_on_objects()' can handle this overlap_filter.clip = None overlap_filter.overlap = None # create object obj = Object(filename='', image=overlap_img, bbox=None, category=None) # add object image to background image merged_img, obj_list = apply_filters_on_objects([obj], overlap_filter, bg_img, 3 * bg_img.shape[1], 3 * bg_img.shape[0]) target_str = 'behind' if overlap_filter.type == 0 else 'in front of' logging.info( " => apply overlap filter %s object with x=%s%%, y=%s%% overlapping and object \'%s\'" % (target_str, x_pct, y_pct * (-1), overlap_img_path)) return merged_img
def list_dir(): res = utils.list_directory(request.args.get('folder', settings.folder)) return jsonify(**res)
def apply_background_filter(background_filter, res_x, res_y, background_path): """ Apply background filter with given background_filter parameters, desired x and y resolution and the path to background directory. Dependent on filter type, the background consists of one given single color, a random color, a desired specified image, a random image out of set standard background directory or a random image out of another given directory. :param background_filter: object containing background parameters :param res_x: x resolution of output background image :param res_y: y resolution of output background image :param background_path: path to set standard background directory :return: bg_img: created background image with desired resolution """ if background_filter.type == 0: # fixed hex color # background_filter.value has format "0x00000000" r_int = utils.hex_to_int(background_filter.value[2:4]) g_int = utils.hex_to_int(background_filter.value[4:6]) b_int = utils.hex_to_int(background_filter.value[6:8]) alpha_int = utils.hex_to_int(background_filter.value[8:10]) bg_img = img_processing.create_colored_background_img( res_x, res_y, r_int, g_int, b_int) logging.info( " => set background to: r: %s, g: %s, b: %s, alpha: %s" % (r_int, g_int, b_int, alpha_int)) elif background_filter.type == 1: # random hex color r_rand = random.randint(0, 255) g_rand = random.randint(0, 255) b_rand = random.randint(0, 255) bg_img = img_processing.create_colored_background_img( res_x, res_y, r_rand, g_rand, b_rand) logging.info(" => set background to random color \'(%s, %s, %s)\'" % (r_rand, g_rand, b_rand)) elif background_filter.type == 2: # background image path specified in background_filter.value bg_img_path = background_filter.value bg_img = img_processing.import_image(bg_img_path) bg_img = img_processing.resize_aspect_ratio_crop_image( bg_img, res_x, res_y) # resize image to the desired resolution logging.info(" => set background to \'%s\'" % bg_img_path) elif background_filter.type == 3: # random background out of backgrounds path bg_img_path_list = utils.list_directory(background_path, file_ext_list=['.jpg', '.png']) random_bg_idx = random.randint(0, len(bg_img_path_list) - 1) bg_img_filename = bg_img_path_list[random_bg_idx] bg_img_path = os.path.abspath(background_path + bg_img_filename) bg_img = img_processing.import_image(bg_img_path) bg_img = img_processing.resize_aspect_ratio_crop_image( bg_img, res_x, res_y) # resize image to the desired resolution logging.info( " => get random background from background path: \'%s\'" % bg_img_path) elif background_filter.type == 4: # random background image from another folder bg_img_path_list = utils.list_directory(background_filter.value, file_ext_list=['.jpg', '.png']) random_bg_idx = random.randint(0, len(bg_img_path_list) - 1) bg_img_filename = bg_img_path_list[random_bg_idx] bg_img_path = os.path.abspath(background_path + bg_img_filename) bg_img = img_processing.import_image(bg_img_path) bg_img = img_processing.resize_aspect_ratio_crop_image( bg_img, res_x, res_y) # resize image to the desired resolution logging.info(" => get random background from another path: \'%s\'" % bg_img_path) else: logging.error("ERROR: Unknown type (%s) of background filter" % background_filter.type) sys.exit(-1) return bg_img