def verify_upscaling_works(self) -> None: """ Verify the upscaler works by upscaling a very small frame, and throws a descriptive error if it doesn't. """ test_file = self.context.workspace + "test_frame.jpg" test_file_upscaled = self.context.workspace + "test_frame_upscaled.jpg" test_frame = Frame() test_frame.create_new(2, 2) test_frame.save_image(test_file) self.upscale_file(test_file, test_file_upscaled) if not file_exists(test_file_upscaled): print( "Your computer could not upscale a test image, which is required for dandere2x to work." ) print( "This may be a hardware issue or a software issue - verify your computer is capable of upscaling " "images using the selected upscaler.") raise Exception("Your computer could not upscale the test file.") os.remove(test_file) os.remove(test_file_upscaled)
def verify_upscaling_works(self) -> None: """ Verify the upscaler works by upscaling a very small frame, and throws a descriptive error if it doesn't. """ test_file = self.context.workspace + "test_frame.jpg" test_file_upscaled = self.context.workspace + "test_frame_upscaled.jpg" test_frame = Frame() test_frame.create_new(2, 2) test_frame.save_image(test_file) self.log.info( "Attempting to upscale file %s into %s to ensure waifu2x is working..." % (test_file, test_file_upscaled)) self.upscale_file(test_file, test_file_upscaled) if not file_exists(test_file_upscaled): self.log.error( "Your computer could not upscale a test image, which is required for dandere2x to work." ) self.log.error( "This may be a hardware issue or a software issue - verify your computer is capable of upscaling " "images using the selected upscaler.") raise Exception("Your computer could not upscale the test file.") self.log.info( "Upscaling *seems* successful. Deleting files and continuing forward. " ) os.remove(test_file) os.remove(test_file_upscaled)
def correct_image(context, frame_base: Frame, list_correction: list): logger = logging.getLogger(__name__) # load context scale_factor = context.scale_factor predictive_vectors = [] out_image = Frame() out_image.create_new(frame_base.width, frame_base.height) out_image.copy_image(frame_base) scale_factor = int(scale_factor) block_size = context.correction_block_size for x in range(int(len(list_correction) / 4)): # / 4 because each there's 4 data points per block predictive_vectors.append( DisplacementVector(int(list_correction[x * 4 + 0]), int(list_correction[x * 4 + 1]), int(list_correction[x * 4 + 2]), int(list_correction[x * 4 + 3]))) # copy over predictive vectors into new image for vector in predictive_vectors: out_image.copy_block(frame_base, block_size * scale_factor, vector.x_2 * scale_factor, vector.y_2 * scale_factor, vector.x_1 * scale_factor, vector.y_1 * scale_factor) return out_image
def make_merge_image(context: Context, frame_inversion: Frame, frame_base: Frame, list_predictive: list, list_differences: list, list_corrections: list, list_fade: list): # Load context logger = logging.getLogger(__name__) out_image = Frame() out_image.create_new(frame_base.width, frame_base.height) # assess the two cases where out images are either duplicates or a new frame completely if not list_predictive and not list_differences: out_image.copy_image(frame_inversion) return out_image if list_predictive and not list_differences: out_image.copy_image(frame_base) return out_image # by copying the image first as the first step, all the predictive elements like # (0,0) -> (0,0) are also coppied out_image.copy_image(frame_base) # run the image through the same plugins IN ORDER it was ran in d2x_cpp out_image = pframe_image(context, out_image, frame_base, frame_inversion, list_differences, list_predictive) out_image = fade_image(context, out_image, list_fade) out_image = correct_image(context, out_image, list_corrections) return out_image
def make_merge_image(context: Context, frame_residual: Frame, frame_previous: Frame, list_predictive: list, list_residual: list, list_corrections: list, list_fade: list): """ This section can best be explained through pictures. A visual way of expressing what 'merging' is doing is this section in the wiki. https://github.com/aka-katto/dandere2x/wiki/How-Dandere2x-Works#part-2-using-observations-to-save-time Inputs: - frame(x) - frame(x+1)_residual - Residual vectors mapping frame(x+1)_residual -> frame(x+1) - Predictive vectors mapping frame(x) -> frame(x+1) Output: - frame(x+1) """ # Load context logger = logging.getLogger(__name__) out_image = Frame() out_image.create_new(frame_previous.width, frame_previous.height) # If list_predictive is empty, then the residual frame is simply the newly # produced image. if not list_predictive: out_image.copy_image(frame_residual) return out_image # By copying the image first as the first step, all the predictive elements of the form (x,y) -> (x,y) # are also copied. This allows us to ignore copying vectors (x,y) -> (x,y), which prevents redundant copying, # thus saving valuable computational time. out_image.copy_image(frame_previous) ################### # Plugins Section # ################### # Note: Run the plugins in the SAME order it was ran in dandere2x_cpp. If not, it won't work correctly. out_image = pframe_image(context, out_image, frame_previous, frame_residual, list_residual, list_predictive) out_image = fade_image(context, out_image, list_fade) out_image = correct_image(context, out_image, list_corrections) return out_image
def debug_image(block_size, frame_base, list_predictive, list_differences, output_location): logger = logging.getLogger(__name__) difference_vectors = [] predictive_vectors = [] out_image = Frame() out_image.create_new(frame_base.width, frame_base.height) out_image.copy_image(frame_base) black_image = Frame() black_image.create_new(frame_base.width, frame_base.height) if not list_predictive and not list_differences: out_image.save_image(output_location) return if list_predictive and not list_differences: out_image.copy_image(frame_base) out_image.save_image(output_location) return # load list into vector displacements for x in range(int(len(list_differences) / 4)): difference_vectors.append( DisplacementVector(int(list_differences[x * 4]), int(list_differences[x * 4 + 1]), int(list_differences[x * 4 + 2]), int(list_differences[x * 4 + 3]))) for x in range(int(len(list_predictive) / 4)): if (int(list_predictive[x * 4 + 0]) != int(list_predictive[x * 4 + 1])) and \ (int(list_predictive[x * 4 + 2]) != int(list_predictive[x * 4 + 3])): predictive_vectors.append( DisplacementVector(int(list_predictive[x * 4 + 0]), int(list_predictive[x * 4 + 1]), int(list_predictive[x * 4 + 2]), int(list_predictive[x * 4 + 3]))) # copy over predictive vectors into new image for vector in difference_vectors: out_image.copy_block(black_image, block_size, vector.x_1, vector.y_1, vector.x_1, vector.y_1) out_image.save_image_quality(output_location, 25)
def make_merge_image(context: Context, frame_residual: Frame, frame_previous: Frame, list_predictive: list, list_residual: list, list_corrections: list, list_fade: list): """ This section can best be explained through pictures. A visual way of expressing what 'merging' is doing is this section in the wiki. https://github.com/aka-katto/dandere2x/wiki/How-Dandere2x-Works#part-2-using-observations-to-save-time Inputs: - frame(x) - frame(x+1)_residual - Residual vectors mapping frame(x+1)_residual -> frame(x+1) - Predictive vectors mapping frame(x) -> frame(x+1) Output: - frame(x+1) """ # Load context logger = logging.getLogger(__name__) out_image = Frame() out_image.create_new(frame_previous.width, frame_previous.height) # If list_predictive and list_predictive are both empty, then the residual frame # is simply the new image. if not list_predictive and not list_predictive: out_image.copy_image(frame_residual) return out_image # by copying the image first as the first step, all the predictive elements like # (0,0) -> (0,0) are also coppied out_image.copy_image(frame_previous) # run the image through the same plugins IN ORDER it was ran in d2x_cpp out_image = pframe_image(context, out_image, frame_previous, frame_residual, list_residual, list_predictive) out_image = fade_image(context, out_image, list_fade) out_image = correct_image(context, out_image, list_corrections) return out_image
def correct_image(context, frame_base: Frame, list_correction: list): """ Try and fix some artifact-residuals by using the same image as reference. Method Tasks: - Load all the vectors for blocks pointing to a block with lower MSE - Apply all the vectors to the image to produce a more 'correct' image """ logger = logging.getLogger(__name__) # load context scale_factor = context.scale_factor out_image = Frame() out_image.create_new(frame_base.width, frame_base.height) out_image.copy_image(frame_base) scale_factor = int(scale_factor) block_size = context.correction_block_size for x in range(int(len(list_correction) / 4)): # / 4 because each there's 4 data points per block # load vector vector = DisplacementVector(int(list_correction[x * 4 + 0]), int(list_correction[x * 4 + 1]), int(list_correction[x * 4 + 2]), int(list_correction[x * 4 + 3])) # apply vector out_image.copy_block(frame_base, block_size * scale_factor, vector.x_2 * scale_factor, vector.y_2 * scale_factor, vector.x_1 * scale_factor, vector.y_1 * scale_factor) return out_image
def make_difference_image(context: Context, raw_frame: Frame, list_difference: list, list_predictive: list): difference_vectors = [] buffer = 5 block_size = context.block_size bleed = context.bleed # first make a 'bleeded' version of input_frame # so we can preform numpy calculations w.o having to catch bleed_frame = raw_frame.create_bleeded_image(buffer) # if there are no items in 'differences' but have list_predictives # then the two frames are identical, so no differences image needed. if not list_difference and list_predictive: out_image = Frame() out_image.create_new(1, 1) return out_image # if there are neither any predictive or inversions # then the frame is a brand new frame with no resemblence to previous frame. # in this case copy the entire frame over if not list_difference and not list_predictive: out_image = Frame() out_image.create_new(raw_frame.width, raw_frame.height) out_image.copy_image(raw_frame) return out_image # turn the list of differences into a list of vectors for x in range(int(len(list_difference) / 4)): difference_vectors.append( DisplacementVector(int(list_difference[x * 4]), int(list_difference[x * 4 + 1]), int(list_difference[x * 4 + 2]), int(list_difference[x * 4 + 3]))) # size of output image is determined based off how many differences there are image_size = int(math.sqrt(len(list_difference) / 4) + 1) * (block_size + bleed * 2) out_image = Frame() out_image.create_new(image_size, image_size) # move every block from the complete frame to the differences frame using vectors. for vector in difference_vectors: out_image.copy_block(bleed_frame, block_size + bleed * 2, vector.x_1 + buffer - bleed, vector.y_1 + buffer + -bleed, vector.x_2 * (block_size + bleed * 2), vector.y_2 * (block_size + bleed * 2)) return out_image
def run(self): self.log.info("Run called.") for x in range(self.start_frame, self.frame_count): # Stop if thread is killed if not self.context.controller.is_alive(): break # Files needed to create a residual image f1 = Frame() f1.load_from_string_controller( self.input_frames_dir + "frame" + str(x + 1) + self.extension_type, self.context.controller) # Load the neccecary lists to compute this iteration of residual making residual_data = get_list_from_file_and_wait( self.residual_data_dir + "residual_" + str(x) + ".txt", self.context.controller) prediction_data = get_list_from_file_and_wait( self.pframe_data_dir + "pframe_" + str(x) + ".txt", self.context.controller) # stop if thread is killed if not self.context.controller.is_alive(): break # Create the output files.. debug_output_file = self.debug_dir + "debug" + str( x + 1) + self.extension_type output_file = self.residual_images_dir + "output_" + get_lexicon_value( 6, x) + ".jpg" # Save to a temp folder so waifu2x-vulkan doesn't try reading it, then move it out_image = self.make_residual_image(self.context, f1, residual_data, prediction_data) if out_image.get_res() == (1, 1): """ If out_image is (1,1) in size, then frame_x and frame_x+1 are identical. We still need to save an outimage for sake of having N output images for N input images, so we save these meaningless files anyways. However, these 1x1 can slow whatever waifu2x implementation down, so we 'cheat' d2x but 'fake' upscaling them, so that they don't need to be processed by waifu2x. """ # Location of the 'fake' upscaled image. out_image = Frame() out_image.create_new(2, 2) output_file = self.residual_upscaled_dir + "output_" + get_lexicon_value( 6, x) + ".png" out_image.save_image(output_file) else: # This image has things to upscale, continue normally out_image.save_image_temp(output_file, self.temp_image) # With this change the wrappers must be modified to not try deleting the non existing residual file if self.context.debug == 1: self.debug_image(self.block_size, f1, prediction_data, residual_data, debug_output_file)
def debug_image(block_size, frame_base, list_predictive, list_differences, output_location): """ Note: I haven't made an effort to maintain this method, as it's only for debugging. This section can best be explained through pictures. A visual way of expressing what 'debug' is doing is this section in the wiki. https://github.com/aka-katto/dandere2x/wiki/How-Dandere2x-Works#part-1-identifying-what-needs-to-be-drawn In other words, this method shows where residuals are, and is useful for finding good settings to use for a video. Inputs: - frame(x) - Residual vectors mapping frame(x)_residual -> frame(x) Output: - frame(x) minus frame(x)_residuals = debug_image """ logger = logging.getLogger(__name__) difference_vectors = [] predictive_vectors = [] out_image = Frame() out_image.create_new(frame_base.width, frame_base.height) out_image.copy_image(frame_base) black_image = Frame() black_image.create_new(frame_base.width, frame_base.height) if not list_predictive and not list_differences: out_image.save_image(output_location) return if list_predictive and not list_differences: out_image.copy_image(frame_base) out_image.save_image(output_location) return # load list into vector displacements for x in range(int(len(list_differences) / 4)): difference_vectors.append( DisplacementVector(int(list_differences[x * 4]), int(list_differences[x * 4 + 1]), int(list_differences[x * 4 + 2]), int(list_differences[x * 4 + 3]))) for x in range(int(len(list_predictive) / 4)): if (int(list_predictive[x * 4 + 0]) != int(list_predictive[x * 4 + 1])) and \ (int(list_predictive[x * 4 + 2]) != int(list_predictive[x * 4 + 3])): predictive_vectors.append( DisplacementVector(int(list_predictive[x * 4 + 0]), int(list_predictive[x * 4 + 1]), int(list_predictive[x * 4 + 2]), int(list_predictive[x * 4 + 3]))) # copy over predictive vectors into new image for vector in difference_vectors: out_image.copy_block(black_image, block_size, vector.x_1, vector.y_1, vector.x_1, vector.y_1) out_image.save_image_quality(output_location, 25)
def make_residual_image(context: Context, raw_frame: Frame, list_residual: list, list_predictive: list): """ This section can best be explained through pictures. A visual way of expressing what 'make_residual_image' is doing is this section in the wiki. https://github.com/aka-katto/dandere2x/wiki/How-Dandere2x-Works#observation_3 Inputs: - frame(x) - Residual vectors mapping frame(x)_residual -> frame(x) Output: - frame(x)_residual """ # Some conditions to check before making a residual image, in both cases, we don't need to do any actual # processing in the function call, if these conditions hold true. if not list_residual and list_predictive: """ If there are no items in 'list_residuals' but have list_predictives then the two frames are identical, so no residual image needed. """ residual_image = Frame() residual_image.create_new(1, 1) return residual_image if not list_residual and not list_predictive: """ If there are neither any predictive or inversions, then the frame is a brand new frame with no resemblence to previous frame. In this case, copy the entire frame over. """ residual_image = Frame() residual_image.create_new(raw_frame.width, raw_frame.height) residual_image.copy_image(raw_frame) return residual_image buffer = 5 block_size = context.block_size bleed = context.bleed """ First make a 'bleeded' version of input_frame, as we need to create a buffer in the event the 'bleed' ends up going out of bounds. In other words, crop the image into an even larger image, so that if if we need to access out of bounds pixels, and place black pixels where it would be out of bounds. """ bleed_frame = raw_frame.create_bleeded_image(buffer) # size of output image is determined based off how many residuals there are image_size = int(math.sqrt(len(list_residual) / 4) + 1) * (block_size + bleed * 2) residual_image = Frame() residual_image.create_new(image_size, image_size) for x in range(int(len(list_residual) / 4)): # load every element in the list into a vector vector = DisplacementVector(int(list_residual[x * 4 + 0]), int(list_residual[x * 4 + 1]), int(list_residual[x * 4 + 2]), int(list_residual[x * 4 + 3])) # apply that vector to the image by copying over their respective blocks. residual_image.copy_block(bleed_frame, block_size + bleed * 2, vector.x_1 + buffer - bleed, vector.y_1 + buffer + -bleed, vector.x_2 * (block_size + bleed * 2), vector.y_2 * (block_size + bleed * 2)) return residual_image
def make_residual_image(context: Context, raw_frame: Frame, list_residual: list, list_predictive: list): """ This section can best be explained through pictures. A visual way of expressing what 'make_residual_image' is doing is this section in the wiki. https://github.com/aka-katto/dandere2x/wiki/How-Dandere2x-Works#observation_3 Inputs: - frame(x) - Residual vectors mapping frame(x)_residual -> frame(x) Output: - frame(x)_residual """ residual_vectors = [] buffer = 5 block_size = context.block_size bleed = context.bleed # first make a 'bleeded' version of input_frame, as we need to create a buffer in the event the 'bleed' # ends up going out of bounds. bleed_frame = raw_frame.create_bleeded_image(buffer) # if there are no items in 'list_residuals' but have list_predictives # then the two frames are identical, so no residual image needed. if not list_residual and list_predictive: out_image = Frame() out_image.create_new(1, 1) return out_image # if there are neither any predictive or inversions # then the frame is a brand new frame with no resemblence to previous frame. # in this case copy the entire frame over if not list_residual and not list_predictive: out_image = Frame() out_image.create_new(raw_frame.width, raw_frame.height) out_image.copy_image(raw_frame) return out_image # turn the list of residuals into a list of vectors for x in range(int(len(list_residual) / 4)): residual_vectors.append( DisplacementVector(int(list_residual[x * 4]), int(list_residual[x * 4 + 1]), int(list_residual[x * 4 + 2]), int(list_residual[x * 4 + 3]))) # size of output image is determined based off how many residuals there are image_size = int(math.sqrt(len(list_residual) / 4) + 1) * (block_size + bleed * 2) out_image = Frame() out_image.create_new(image_size, image_size) # move every block from the complete frame to the residual frame using vectors. for vector in residual_vectors: out_image.copy_block(bleed_frame, block_size + bleed * 2, vector.x_1 + buffer - bleed, vector.y_1 + buffer + -bleed, vector.x_2 * (block_size + bleed * 2), vector.y_2 * (block_size + bleed * 2)) return out_image
def residual_loop(context): """ Call the 'make_residual_image' method for every image that needs to be made into a residual. Method Tasks: - Load and wait for the files needed to create a residual image. - Call 'make_residual_image' once the needed files exist """ # load variables from context workspace = context.workspace residual_upscaled_dir = context.residual_upscaled_dir residual_images_dir = context.residual_images_dir residual_data_dir = context.residual_data_dir pframe_data_dir = context.pframe_data_dir input_frames_dir = context.input_frames_dir frame_count = context.frame_count block_size = context.block_size extension_type = context.extension_type debug_dir = context.debug_dir debug = context.debug temp_image = context.temp_image_folder + "tempimage.jpg" logger = logging.getLogger(__name__) logger.info((workspace, 1, frame_count, block_size)) # for every frame in the video, create a residual_frame given the text files. for x in range(1, frame_count): f1 = Frame() f1.load_from_string_wait(input_frames_dir + "frame" + str(x + 1) + extension_type) # Load the neccecary lists to compute this iteration of residual making residual_data = get_list_from_file(residual_data_dir + "residual_" + str(x) + ".txt") prediction_data = get_list_from_file(pframe_data_dir + "pframe_" + str(x) + ".txt") # Create the output files.. debug_output_file = debug_dir + "debug" + str(x + 1) + extension_type output_file = residual_images_dir + "output_" + get_lexicon_value( 6, x) + ".jpg" # Save to a temp folder so waifu2x-vulkan doesn't try reading it, then move it out_image = make_residual_image(context, f1, residual_data, prediction_data) if out_image.get_res() == (1, 1): """ If out_image is (1,1) in size, then frame_x and frame_x+1 are identical. We still need to save an outimage for sake of having N output images for N input images, so we save these meaningless files anyways. However, these 1x1 can slow whatever waifu2x implementation down, so we 'cheat' d2x but 'fake' upscaling them, so that they don't need to be processed by waifu2x. """ # Location of the 'fake' upscaled image. out_image = Frame() out_image.create_new(2, 2) output_file = residual_upscaled_dir + "output_" + get_lexicon_value( 6, x) + ".png" out_image.save_image(output_file) else: # This image has things to upscale, continue normally out_image.save_image_temp(output_file, temp_image) # With this change the wrappers must be modified to not try deleting the non existing residual file if debug == 1: debug_image(block_size, f1, prediction_data, residual_data, debug_output_file)