def _load_viewer(config: Dict) -> Viewer: ''' Loads a Viewer from the given Viewer configuration. Args: config: Viewer configuration. Returns: Viewer instance as specified by the Viewer configuration. ''' assert 'Type' in config, 'Scope "Viewer" in configuration file is missing key "Type".' if config['Type'] == 'Orthographic': assert 'Direction' in config, 'Scope "Viewer" in configuration file is missing key "Direction".' return OrthographicViewer( direction=torch.tensor(config['Direction'], dtype=torch.float, device=utils.get_device_name())) elif config['Type'] == 'Perspective': assert 'Position' in config, 'Scope "Viewer" in configuration file is missing key "Position".' return PerspectiveViewer( position=torch.tensor(config['Position'], dtype=torch.float, device=utils.get_device_name())) else: raise ValueError('Viewer type "%s" is not supported.' % config['Type'])
def _load_camera(config: Dict) -> Camera: ''' Loads a Camera from the given Camera configuration. Args: config: Camera configuration. Returns: Camera instance as specified by the Camera configuration. ''' assert 'Type' in config, 'Scope "Camera" in configuration file is missing key "Type".' if config['Type'] == 'Identity': return IdentityCamera() if config['Type'] == 'Perspective': for key in ('Position', 'Direction', 'Field of View', 'Resolution', 'Exposure'): assert key in config, f'Scope "Camera" in configuration file is missing key "{key}".' return PerspectiveCamera( position=torch.tensor(config['Position'], dtype=torch.float, device=utils.get_device_name()), direction=torch.tensor(config['Direction'], dtype=torch.float, device=utils.get_device_name()), field_of_view=torch.tensor(config['Field of View'], dtype=torch.float, device=utils.get_device_name()), resolution=torch.tensor(config['Resolution'], device=utils.get_device_name()), exposure=config['Exposure']) else: raise ValueError('Camera type "%s" is not supported.' % config['Type'])
def _load_network(path: str) -> SVBRDFAutoencoder: ''' Loads an SVBRDFAutoencoder from the given SVBRDFAutoencoder configuration. Args: path: Path to the SVBRDFAutoencoder YAML configuration. Returns: SVBRDFAutoencoder instance as specified by the SVBRDFAutoencoder configuration. ''' with open(path, 'r') as file: config = yaml.safe_load(file) for key in ('Dimensions', 'Parameters', 'Encoders', 'Decoder'): assert key in config, f'Scope "root" in configuration file is missing key "{key}".' for key in ('Path', 'Load'): assert key in config[ 'Parameters'], f'Scope "Parameters" in configuration file is missing key "{key}".' path = config['Parameters']['Path'] load = config['Parameters']['Load'] dims = config['Dimensions'] encoders = {} for key in ('Local', 'Global', 'Periodic'): encoders[key] = Configuration._load_subnetwork( config=config['Encoders'][key]) decoder = Configuration._load_subnetwork(config=config['Decoder']) device = utils.get_device_name() autoencoder = SVBRDFAutoencoder(dims=dims, path=path, encoders=encoders, decoder=decoder).to(device) if load: autoencoder.load() return autoencoder
def index(): """Video streaming""" # now = datetime.now() # timeString = now.strftime("%Y-%m-%d %H:%M") register_url = get_device_registration_url() with requests.Session() as s: data = { 'DeviceName': get_device_name(), 'DateCreated': get_current_time() } headers = {"Content-Type": "application/json"} r = s.post(register_url, headers=headers, data=data) json_response = r.json() print("json response == " + json_response["data"]["ipAddress"]) session['ip_address'] = json_response["data"]["ipAddress"] templateData = { 'title': 'HELLO!', # 'time': timeString, 'time': "2020-03-06", 'response': json_response, 'ipaddress': json_response["data"]["ipAddress"] } print(session) return render_template('index.html', **templateData)
def load(self) -> None: ''' Loads the parameters of this SVBRDFAutoencoder. ''' device = utils.get_device_name() self.load_state_dict( torch.load(self._path, map_location=torch.device(device))) logging.debug('Loaded network parameters from "%s"', self._path)
def set_from_file(self, file): self.file = file self.connected = True devfile = open(file, "rb") self.name = U.get_device_name(devfile) self.axis_map, self.axis_states = U.get_joystick_axis_map(devfile) self.button_map, self.button_states = U.get_joystick_button_map(devfile) self.watch_id = GObject.io_add_watch(devfile, GObject.IO_IN, self._watch_cb)
def __init__(self, path: str, num_samples: int, intensity: float) -> None: ''' Constructs a new ImageLight from the given image path, number of samples, and intensity. Args: path: Path to an environment map image. num_samples: Number of samples to take from the environment map. intensity: Scalar applied to each environment map radiance sample. ''' assert path, "Path cannot be empty or set to None." assert num_samples > 0, "Number of samples must be greater than zero." self._image = image.load(path, 'sRGB') self._intensity = intensity # Construct a grid of azimuth (ϕ) and zenith (θ) angles uniformly distributed over the upper unit hemisphere. self._samples = torch.stack([ 2 * math.pi * torch.rand(num_samples, device=utils.get_device_name()), torch.acos(torch.rand(num_samples, device=utils.get_device_name())) ], dim=1)
def _load_light(config: Dict) -> Light: ''' Loads a Light from the given Light configuration. Args: config: Light configuration. Returns: Light instance as specified by the Light configuration. ''' assert 'Type' in config, 'Scope "Light" in configuration file is missing key "Type".' if config['Type'] == 'Directional': for key in ('Direction', 'Lumens'): assert key in config, f'Scope "Light" in configuration file is missing key "{key}".' return DirectionalLight( direction=torch.tensor(config['Direction'], dtype=torch.float, device=utils.get_device_name()), lumens=torch.tensor(config['Lumens'], dtype=torch.float, device=utils.get_device_name())) elif config['Type'] == 'Punctual': for key in ('Position', 'Lumens'): assert key in config, f'Scope "Light" in configuration file is missing key "{key}".' return PunctualLight( position=torch.tensor(config['Position'], dtype=torch.float, device=utils.get_device_name()), lumens=torch.tensor(config['Lumens'], dtype=torch.float, device=utils.get_device_name())) elif config['Type'] == 'Image': for key in ('Path', 'Samples', 'Intensity'): assert key in config, f'Scope "Light" in configuration file is missing key "{key}".' return ImageLight(path=config['Path'], num_samples=config['Samples'], intensity=config['Intensity']) else: raise ValueError('Light type "%s" is not supported.' % config['Type'])
def apply(self, normals: Tensor, parameters: Tensor, lights: List[Light], viewer: Viewer) -> Tuple[Tensor, Tensor, List[Light], Viewer]: '''See Transform.apply().''' replaced_parameters = parameters.clone().detach() for i, target in enumerate(self._targets): index = target['Index'] min_value = target['Min Value'] max_value = target['Max Value'] uniform_random_field = torch.rand_like( parameters[:, :, :, index], device=utils.get_device_name()) replaced_parameters[:, :, :, index] = uniform_random_field * ( max_value - min_value) + min_value return normals, replaced_parameters, lights, viewer
def radiance(self, points: Tensor) -> Tensor: '''See Light.radiance().''' num_samples = self.samples.size(0) # The bilinear sampling function operates over [N, C, H, W] Tensors. sample_input = self._image.permute(2, 0, 1).unsqueeze(0) # Convert the sampled angles into environment map coordinates. sample_grid = torch.zeros(size=(1, 1, num_samples, 2), device=utils.get_device_name()) sample_grid[0, 0, :, 0] = self.samples[:, 0] / math.pi - 1 sample_grid[0, 0, :, 1] = self.samples[:, 1] / math.pi * 2 - 1 # Return the incident radiance from the environment map. incident_radiance = torch.nn.functional.grid_sample( sample_input, sample_grid, mode='bilinear', align_corners=False).permute(0, 2, 3, 1) return self.intensity / num_samples * incident_radiance.view( -1, 3).expand(points.size(0), points.size(1), -1, 3)
def compute_radiance(network_normals: Tensor, network_svbrdf: SVBRDF, dataset_normals: Tensor, dataset_svbrdf: SVBRDF) -> Tuple[Tensor, Tensor]: ''' Computes the radiance from the given normals and SVBRDFs with respect to a random point Light and Viewer. Args: network_normals: Tensor [B, R, C, 3] of SVBRDF autoencoder normals. network_svbrdf: SVBRDF with embedded SVBRDF autoencoder parameters. dataset_normals: Tensor [B, R, C, 3] of ground-truth normals. dataset_svbrdf: SVBRDF with embedded ground-truth parameters. Returns: Tuple containing the SVBRDF autoencoder and Dataset radiance Tensors. ''' # There is no harm in sharing the same drawing canvas for both the network and dataset renderings. texture_rows = dataset_normals.size(1) texture_cols = dataset_normals.size(2) surface = utils.create_grid(num_rows=texture_rows, num_cols=texture_cols) # The Light and Viewer are sampled from a cosine-weighted distribution following the Single-Image SVBRDF Capture # with a Rendering-Aware Deep Network paper. origin = torch.tensor([0.5, 0.5, 0.0], device=utils.get_device_name()) lights = [ PunctualLight(position=utils.sample_cosine_hemisphere(origin), lumens=torch.rand(1).expand(3) * 2 + 0.5) ] viewer = PerspectiveViewer(position=utils.sample_cosine_hemisphere(origin)) network_radiance = shader.shade(surface=surface, normals=network_normals, lights=lights, viewer=viewer, svbrdf=network_svbrdf) dataset_radiance = shader.shade(surface=surface, normals=dataset_normals, lights=lights, viewer=viewer, svbrdf=dataset_svbrdf) return network_radiance, dataset_radiance
def apply(self, normals: Tensor, parameters: Tensor, lights: List[Light], viewer: Viewer) -> Tuple[Tensor, Tensor, List[Light], Viewer]: '''See Transform.apply().''' device = utils.get_device_name() scalar = torch.rand(1) * (self._max_scalar - self._min_scalar) + self._min_scalar # Take care not to modify the original list of Lights. elevated_lights = [light for light in lights] for i, light in enumerate(lights): # Only punctual Lights can be raised or lowered. if isinstance(light, PunctualLight): elevated_position = light.position * torch.tensor( [1.0, 1.0, scalar], device=device) elevated_lights[i] = PunctualLight(position=elevated_position, lumens=light.lumens) elevated_viewer = viewer # Similarly, only perspective Viewers can be raised or lowered. if isinstance(viewer, PerspectiveViewer): elevated_position = viewer.position * torch.tensor( [1.0, 1.0, scalar], device=device) elevated_viewer = PerspectiveViewer(position=elevated_position) return normals, parameters, elevated_lights, elevated_viewer
def load(path: str, encoding: str = 'RGB') -> Tensor: ''' Loads the image at the given path using the supplied encoding. Args: path: Path to the image. encoding: Encoding of the image. Returns: Tensor [R, C, X] representing the normalized pixel values in the image. ''' assert path, "Path cannot be empty or set to None." array = imageio.imread(path) device = utils.get_device_name() image = torchvision.transforms.ToTensor()(array).to(device).permute( 1, 2, 0)[:, :, :3] if encoding == 'sRGB': image = convert_sRGB_to_RGB(image) elif encoding == 'Greyscale': image = convert_RGB_to_greyscale(image) elif encoding != 'RGB': raise Exception(f'Image encoding "{encoding}" is not supported."') logging.debug('Loaded image from "%s"', path) return image
def _merge_flow(config: Configuration) -> None: ''' The "merge" flow melds two overlapping textures by smoothly blending their latent fields. Args: config: Configuration specifying the parameters of the flow. ''' with torch.no_grad(): autoencoder, svbrdf, lights, viewer, camera, overlap, input_paths, output_path = config.load_merge_flow( ) autoencoder.eval() # It is assumed that the dimensions of the input images will be accepted by the network. input_images = torch.stack([ image.load(path=input_path, encoding='sRGB') for input_path in input_paths ], dim=0) # The radial distance field should be the same for both input images. num_texture_rows = autoencoder.dimensions['Texture']['Input'][0] num_texture_cols = autoencoder.dimensions['Texture']['Input'][1] input_distance = utils.create_radial_distance_field( num_rows=num_texture_rows, num_cols=num_texture_cols) # By convention, PyTorch expects Tensors to be in [B, D, R, C] format. input_batch = torch.cat( [input_images, input_distance.expand(2, -1, -1, -1)], dim=3).permute(0, 3, 1, 2) # The width and height of the SVBRDF autoencoder latent are shared between all latent components. num_latent_rows = autoencoder.dimensions['Latent']['Local'][0] num_latent_cols = autoencoder.dimensions['Latent']['Local'][1] # The latent field corresponding to each texture must be padded in the region where it has no influence. device = utils.get_device_name() channels = { key: autoencoder.dimensions['Latent'][key][2] for key in ('Local', 'Global', 'Periodic') } padding = torch.zeros( (num_latent_rows, num_latent_cols - overlap, sum( channels.values())), device=device) # The latent field is blended smoothly across the overlapping region as follows: # +------------+---------------------+------------+ # | α = 0.00 | α = 0.00 ... 1.00 | α = 1.00 | # +------------+---------------------+------------+ # <----- Overlap -----> texture_latents = autoencoder.encode(input_batch).permute(0, 2, 3, 1) widened_latents = torch.stack([ torch.cat([texture_latents[0], padding], dim=1), torch.cat([padding, texture_latents[1]], dim=1) ], dim=0) alphas = torch.cat([ torch.zeros(num_latent_cols - overlap, device=device), torch.linspace(0, 1, overlap, device=device), torch.ones(num_latent_cols - overlap, device=device) ]).expand(num_latent_rows, -1).unsqueeze(-1) blended_latents = torch.lerp(widened_latents[0], widened_latents[1], alphas).permute(2, 0, 1) # The periodic component should be replaced to be consistent with the blended global field.. global_field = blended_latents[channels['Local']:channels['Local'] + channels['Global'], :, :] blended_latents[ -channels['Periodic']:, :, :] = autoencoder.derive_periodic_field( global_field.unsqueeze(0)).squeeze(0) # The fully-convolutional nature of the SVBRDF decoder trivializes the creation of textures with arbitrary sizes. normals, svbrdf.parameters = SVBRDFAutoencoder.interpret( autoencoder.decode(blended_latents.unsqueeze(0))) _shade_render_save(normals=normals, svbrdf=svbrdf, lights=lights, viewer=viewer, camera=camera, path=output_path)
def _shuffle_flow(config: Configuration) -> None: ''' The "shuffle" flow expands the SVBRDF parameters of an image to fill an arbitrary plane by shuffling latent tiles. Args: config: Configuration specifying the parameters of the flow. ''' with torch.no_grad(): autoencoder, svbrdf, lights, viewer, camera, tile_size, output_size, input_path, output_path = config.load_shuffle_flow( ) autoencoder.eval() # Continuing to index sizes with 0 and 1 is simultaneously confusing and a potential debugging nightmare. num_tile_rows, num_tile_cols = tile_size num_output_rows, num_output_cols = output_size # Similarly, it is worthwhile to give names to the otherwise-generic SVBRDF autoencoder dimensions. num_latent_rows = autoencoder.dimensions['Latent']['Local'][0] num_latent_cols = autoencoder.dimensions['Latent']['Local'][1] num_texture_rows = autoencoder.dimensions['Texture']['Input'][0] num_texture_cols = autoencoder.dimensions['Texture']['Input'][1] row_expansion_ratio = autoencoder.dimensions['Texture']['Output'][ 0] // num_latent_rows col_expansion_ratio = autoencoder.dimensions['Texture']['Output'][ 1] // num_latent_cols # These sanity checks may seem obvious but you never know... assert num_tile_rows <= num_latent_rows, 'Tile height cannot exceed the height of the latent field.' assert num_tile_cols <= num_latent_cols, 'Tile width cannot exceed the width of the latent field.' assert num_output_rows % ( row_expansion_ratio * num_tile_rows ) == 0, 'Latent height inferred from the output height must be a multiple of the tile height.' assert num_output_cols % ( col_expansion_ratio * num_tile_cols ) == 0, 'Latent width inferred from the output width must be a multiple of the tile width.' # It is assumed that the dimensions of the input images will be accepted by the network. input_images = image.load(path=input_path, encoding='sRGB').unsqueeze(0) input_distance = utils.create_radial_distance_field( num_rows=num_texture_rows, num_cols=num_texture_cols).unsqueeze(0) input_batch = torch.cat([input_images, input_distance], dim=3).permute(0, 3, 1, 2) input_latent = autoencoder.encode(input_batch) # As mentioned in the assertions, the size of the shuffled latent field can be inferred from the desired output texture size. num_shuffled_rows = num_output_rows // row_expansion_ratio num_shuffled_cols = num_output_cols // col_expansion_ratio shuffled_latent = torch.zeros( (1, input_latent.size(1), num_shuffled_rows, num_shuffled_cols), device=utils.get_device_name()) # The shuffled latent is populated with random tiles from the input image latent. for row in range(0, shuffled_latent.size(2), num_tile_rows): for col in range(0, shuffled_latent.size(3), num_tile_cols): original_row_crop, original_col_crop = utils.sample_embedded_rectangle( num_outer_rows=input_latent.size(2), num_inner_rows=num_tile_rows, num_outer_cols=input_latent.size(3), num_inner_cols=num_tile_cols) shuffled_row_crop, shuffled_col_crop = slice( row, row + num_tile_rows), slice(col, col + num_tile_cols) shuffled_latent[:, :, shuffled_row_crop, shuffled_col_crop] = input_latent[:, :, original_row_crop, original_col_crop] # The periodic latent component needs to be aligned with its relative position in the field. channels = { key: autoencoder.dimensions['Latent'][key][2] for key in ('Local', 'Global', 'Periodic') } global_field = shuffled_latent[:, channels['Local']:channels['Local'] + channels['Global'], :, :] shuffled_latent[:, -channels[ 'Periodic']:, :, :] = autoencoder.derive_periodic_field( global_field) # The fully-convolutional nature of the SVBRDF decoder trivializes the creation of textures with arbitrary sizes. normals, svbrdf.parameters = SVBRDFAutoencoder.interpret( autoencoder.decode(shuffled_latent)) _shade_render_save(normals=normals, svbrdf=svbrdf, lights=lights, viewer=viewer, camera=camera, path=output_path)
def load(self, file_name=None): if file_name is None: file_name = self.get_file_name() path = os.path.join(self.model_dir, file_name) self.load_state_dict(torch.load(path, map_location=utils.get_device_name())) print(f"Loaded model state dict from {path}")
def optimize( autoencoder: SVBRDFAutoencoder, svbrdf: SVBRDF, datasets: Dict[str, Dataset], optimizer: torch.optim.Optimizer, # type: ignore epochs: int, cycles: int, samples: int, frequencies: Dict[str, int], loss_weights: Weights, early_stopping: Dict, experiment: str) -> None: ''' Optimizes the given SVBRDF autoencoder using the provided SVBRDF, Datasets, Optimizer, and hyperparameters. Args: autoencoder: SVBRDFAutoencoder to be optimized. svbrdf: SVBRDF intended for the output of the SVBRDF autoencoder. datasets: Mapping between Dataset names (e.g., "Training") and, well, Datasets. optimizer: Optimizer that updates the parameters in the SVBRDF autoencoder. epochs: Number of training epochs. cycles: Number of training steps to execute during each epoch. samples: Size of a training batch. frequencies: Mapping between event names (e.g., "Parameter Checkpoint") and the number of training steps between executions of these events. loss_weights: Mapping between loss types (e.g., "Reconstruction") and dictionaries that associate loss components (e.g., "Style") with their corresponding weights. experiment: Name of the current experiment. ''' # The structure of the given dictionaries are checked here rather than the Configuration to avoid bloating distant # code and keep the relevant implementation in one place. for key in ('Training', 'Testing'): assert key in datasets, f'Dataset dictionary is missing key {key}.' for key in ('Reconstruction', ): assert key in loss_weights, f'Loss weights dictionary is missing key {key}.' # for key in ('Content', 'Style'): # assert key in loss_weights['Diversity'], f'Loss weights dictionary is missing key {key} under scope "Diversity".' for key in ('Content', 'Style', 'Texel'): assert key in loss_weights[ 'Reconstruction'], f'Loss weights dictionary is missing key {key} under scope "Reconstruction".' for key in ('Tests Publication', 'Image Publication', 'Parameter Checkpoint'): assert key in frequencies, f'Frequencies dictionary is missing key {key}.' for key in ('Epsilon', 'Patience'): assert key in early_stopping, f'Early stopping dictionary is missing key {key}.' def zero_backward_step(loss: Tensor) -> None: '''Updates the parameters of the SVBRDF autoencoder using the given loss Tensor, taking care to clear gradients beforehand.''' optimizer.zero_grad() loss.backward() optimizer.step() # Sharing the same VGG-19 network instance across all invocations is absolutely critical to performance. # Furthermore, the VGG-19 network must be initialized after the primary computation device has been set. vgg19 = VGG19().to(utils.get_device_name()) vgg19.eval() # A SummaryWriter can be used to publish data to TensorBoard. run_name = datetime.datetime.now().strftime(f'%Y-%m-%d_%H-%M-%S') dashboard = SummaryWriter(f'results/runs/{run_name} - {experiment}') # Tracking the number of training steps is helpful for TensorBoard tracking and implementing "every-X-step" behaviour. steps = 0 # The best testing loss record tracks the minimum testing loss achieved so far. best_testing_loss = torch.tensor(float('inf')) # The early stopping counter tracks the number of early stopping checks that have transpired since the best testing loss was updated. early_stopping_counter = 0 # Each epoch introduces a new material to the training loop. for epoch in range(epochs): for cycle in tqdm.tqdm(range(cycles), desc=f'Epoch {epoch} Progress', total=cycles): # Materials from the Dataset are selected in a round-robin fashion following the training technique described # in the Diversified Texture Synthesis with Feed-forward Networks paper. material = cycle % min(epoch + 1, len(datasets['Training'])) # The reconstruction loss encourages the network to accurately infer the SVBRDF parameters of a given texture. reconstruction_losses = compute_reconstruction_losses( autoencoder=autoencoder, network_svbrdf=svbrdf, samples=samples, dataset=datasets['Training'], material=material, vgg19=vgg19) reconstruction_losses['Total'] = loss_weights['Reconstruction']['Content'] * reconstruction_losses['Content'] + \ loss_weights['Reconstruction']['Style'] * reconstruction_losses['Style'] + \ loss_weights['Reconstruction']['Texel'] * reconstruction_losses['Texel'] zero_backward_step(loss=reconstruction_losses['Total']) # The diversity loss encourages the network to encode the style of a texture in the global latent vector. # diversity_losses = compute_diversity_losses(autoencoder=autoencoder, network_svbrdf=svbrdf, samples=samples, # dataset=datasets['Training'], material=material, vgg19=vgg19) # diversity_losses['Total'] = loss_weights['Diversity']['Content'] * diversity_losses['Content'] + \ # loss_weights['Diversity']['Style'] * diversity_losses['Style'] # zero_backward_step(loss=diversity_losses['Total']) # The training losses are published to the dashboard after each training iteration. texture = datasets['Training'].textures[material] losses = {'Reconstruction': reconstruction_losses} publish_scalar_results(dashboard=dashboard, mode='Training', steps=steps, texture=texture, losses=losses) # Adding one to the number of steps avoids triggering an event on the first training iteration. progress = steps + 1 if progress % frequencies['Parameter Checkpoint'] == 0: autoencoder.save() if progress % frequencies['Image Publication'] == 0: materials = range(min(epoch + 1, len(datasets['Training']))) publish_image_results(dashboard=dashboard, mode='Training', steps=steps, autoencoder=autoencoder, network_svbrdf=svbrdf, dataset=datasets['Training'], materials=materials) if progress % frequencies['Tests Publication'] == 0: publish_testing_results(dashboard=dashboard, steps=steps, autoencoder=autoencoder, network_svbrdf=svbrdf, dataset=datasets['Testing'], samples=samples, loss_weights=loss_weights, vgg19=vgg19) if progress % frequencies['Early Stopping'] == 0 and epoch >= len( datasets['Training']): # The testing loss is the mean loss of each material in the testing dataset. testing_loss = compute_testing_loss( autoencoder=autoencoder, network_svbrdf=svbrdf, dataset=datasets['Testing'], samples=samples, loss_weights=loss_weights, vgg19=vgg19) # The epsilon factor avoids delaying the early stopping due to noise. if testing_loss < best_testing_loss - early_stopping['Epsilon']: best_testing_loss = testing_loss early_stopping_counter = 0 else: early_stopping_counter += 1 if early_stopping_counter >= early_stopping['Patience']: logging.info( 'Early stopping triggered at epoch %d cycle %d', epoch, cycle) # The parameter weights may no longer be "optimal" but they are probably close enough. autoencoder.save() dashboard.close() return steps += 1 dashboard.close()
def _morph_flow(config: Configuration) -> None: ''' The "morph" flow morphs one texture into another over a series of discrete tiles. Args: config: Configuration specifying the parameters of the flow. ''' with torch.no_grad(): autoencoder, svbrdf, lights, viewer, camera, between, input_paths, output_path = config.load_morph_flow( ) autoencoder.eval() # The total number of tiles includes the two textures on either end as well as the tiles between the textures. tiles = 2 + between device = utils.get_device_name() # It is assumed that the dimensions of the input images will be accepted by the network. input_images = torch.stack([ image.load(path=input_path, encoding='sRGB') for input_path in input_paths ], dim=0) # The radial distance field should be the same for both input images. num_texture_rows = autoencoder.dimensions['Texture']['Input'][0] num_texture_cols = autoencoder.dimensions['Texture']['Input'][1] input_distance = utils.create_radial_distance_field( num_rows=num_texture_rows, num_cols=num_texture_cols) # By convention, PyTorch expects Tensors to be in [B, D, R, C] format. input_batch = torch.cat( [input_images, input_distance.expand(2, -1, -1, -1)], dim=3).permute(0, 3, 1, 2) # The width and height of the SVBRDF autoencoder latent are shared between all latent components. num_latent_rows = autoencoder.dimensions['Latent']['Local'][0] num_latent_cols = autoencoder.dimensions['Latent']['Local'][1] # The local field latent is blended such that each texel within a tile has the same alpha component. # +------------+------------+------------+------------+------------+ # | α = 0.00 | α = 0.25 | α = 0.50 | α = 0.75 | α = 1.00 | # +------------+------------+------------+------------+------------+ local_encoder_output = autoencoder.encoders['Local'].forward( input_batch) local_field_output = local_encoder_output.repeat(1, 1, 1, tiles).permute( 0, 2, 3, 1) local_field_alphas = torch.linspace( 0, 1, tiles, device=device).repeat_interleave(num_latent_cols).expand( num_latent_rows, -1).unsqueeze(-1) local_field = torch.lerp(local_field_output[0], local_field_output[1], local_field_alphas).permute(2, 0, 1) # The global field latent is blended continuously between the left and right textures. # +------------+------------+------------+------------+------------+ # | α = 0.00 | α = 0.00 ... ... 0.50 ... ... 1.00 | α = 1.00 | # +------------+------------+------------+------------+------------+ global_encoder_output = autoencoder.encoders['Global'].forward( input_batch) global_field_output = global_encoder_output.expand( num_latent_rows, num_latent_cols * tiles, -1, -1).permute(2, 0, 1, 3) global_field_alphas = torch.cat([ torch.zeros(num_latent_cols, device=device), torch.linspace(0, 1, num_latent_cols * between, device=device), torch.ones(num_latent_cols, device=device) ]).expand(num_latent_rows, -1).unsqueeze(-1) global_field = torch.lerp(global_field_output[0], global_field_output[1], global_field_alphas).permute(2, 0, 1) # Fortunately, the periodic field latent does not demand any special treatment. periodic_field = autoencoder.derive_periodic_field( global_field.unsqueeze(0)).squeeze(0) # The fully-convolutional nature of the SVBRDF decoder trivializes the creation of textures with arbitrary sizes. latents = torch.cat([local_field, global_field, periodic_field], dim=0).unsqueeze(0) normals, svbrdf.parameters = SVBRDFAutoencoder.interpret( autoencoder.decode(latents)) _shade_render_save(normals=normals, svbrdf=svbrdf, lights=lights, viewer=viewer, camera=camera, path=output_path)
def evaluate(self, normals: Tensor, incident_directions: Tensor, outbound_directions: Tensor) -> Tensor: '''See SVBRDF.evaluate().''' base_colours, subsurface, metallic, specular_amounts, specular_tints, roughness, anisotropy_levels, anisotropy_angles, \ sheen_amounts, sheen_tints, clearcoat_amounts, clearcoat_gloss = torch.split(self.parameters, [3] + [1] * 11, dim=-1) # The incident and outbound halfway zeniths are shared across several SVBRDF lobes. halfways = vector.normalize(incident_directions + outbound_directions) halfway_zeniths = vector.dot(incident_directions, halfways) # Isolate the hue and saturation from the base colour luminosity: https://www.w3.org/Graphics/Color/sRGB. dtype = self.parameters.dtype device = utils.get_device_name() luminosity = torch.sum( 1E-5 + base_colours * torch.tensor([0.2126, 0.7152, 0.0722], dtype=dtype, device=device), dim=-1, keepdim=True) tint_colours = base_colours / luminosity # Compute the Lambertian SVBRDF contribution with two Fresnel factors. diffuse_lambertian_colours = 0.5 + 2 * torch.unsqueeze( roughness, dim=3) * halfway_zeniths**2 diffuse_lambertian_fresnel_factors = MicrofacetSVBRDF.F_Disney(diffuse_lambertian_colours, normals, incident_directions) * \ MicrofacetSVBRDF.F_Disney(diffuse_lambertian_colours, normals, outbound_directions) diffuse_lambertian_terms = LambertianSVBRDF(base_colours).evaluate(normals, incident_directions, outbound_directions) \ * diffuse_lambertian_fresnel_factors incident_zeniths = vector.dot(incident_directions, normals).clamp(1E-5, 1) outbound_zeniths = vector.dot(outbound_directions, normals).clamp(1E-5, 1) # Compute the Hanrahan-Krueger-inspired SVBRDF contribution (to model subsurface scattering). diffuse_subsurface_colours = torch.unsqueeze( roughness, dim=3) * halfway_zeniths**2 diffuse_subsurface_fresnel_factors = MicrofacetSVBRDF.F_Disney(diffuse_subsurface_colours, normals, incident_directions) * \ MicrofacetSVBRDF.F_Disney(diffuse_subsurface_colours, normals, outbound_directions) diffuse_subsurface_terms = LambertianSVBRDF(base_colours).evaluate(normals, incident_directions, outbound_directions) \ * 1.25 * (diffuse_subsurface_fresnel_factors * (1 / (incident_zeniths + outbound_zeniths) - 0.5) + 0.5) # Compute the sheen SVBRDF contribution (for cloth materials). diffuse_sheen_colours = torch.lerp( torch.tensor(1.0, dtype=dtype, device=device), tint_colours, sheen_tints) diffuse_sheen_terms = torch.unsqueeze( sheen_amounts * diffuse_sheen_colours, dim=3) * (1 - halfway_zeniths)**5 # Compute the primary specular SVBRDF contribution using an anisotropic GGX microfacet model. specular_aspects = torch.sqrt(1.0 - 0.9 * anisotropy_levels) specular_alphas_x = torch.max( roughness**2 / specular_aspects, torch.tensor([1E-3], dtype=dtype, device=device)) specular_alphas_y = torch.max( roughness**2 * specular_aspects, torch.tensor([1E-3], dtype=dtype, device=device)) specular_colours = torch.lerp( specular_amounts * 0.08 * torch.lerp(torch.tensor(1.0, dtype=dtype, device=device), tint_colours, specular_tints), base_colours, metallic) specular_parameters = torch.cat([ specular_alphas_x, specular_alphas_y, 2 * math.pi * anisotropy_angles, specular_colours ], dim=3) specular_terms = MicrofacetSVBRDF( D=(MicrofacetSVBRDF.D_GGX_Anisotropic, [0, 1, 2]), F=(MicrofacetSVBRDF.F_Schlick, [3, 4, 5]), G=(MicrofacetSVBRDF.G_GGX_Anisotropic, [0, 1, 2]), parameters=specular_parameters).evaluate(normals, incident_directions, outbound_directions) # Compute the clearcoat SVBRDF contribution using a GTR model where γ=1. clearcoat_D_alphas = torch.lerp( torch.full(clearcoat_gloss.shape, 0.1, dtype=dtype, device=device), torch.full(clearcoat_gloss.shape, 0.001, dtype=dtype, device=device), clearcoat_gloss) clearcoat_G_alphas = torch.full(clearcoat_D_alphas.shape, 0.25, dtype=dtype, device=device) clearcoat_colours = torch.full(base_colours.shape, 0.04, dtype=dtype, device=device) clearcoat_parameters = torch.cat( [clearcoat_D_alphas, clearcoat_G_alphas, clearcoat_colours], dim=3) clearcoat_terms = MicrofacetSVBRDF( D=(MicrofacetSVBRDF.D_Berry, [0]), F=(MicrofacetSVBRDF.F_Schlick, [2, 3, 4]), G=(MicrofacetSVBRDF.G_GGX_Isotropic, [1]), parameters=clearcoat_parameters).evaluate(normals, incident_directions, outbound_directions) # Cull any reflections that pass through the surface. visible = (incident_zeniths >= 0) & (outbound_zeniths >= 0) # Combine the SVBRDF contributions in a principled way. diffuse_terms = diffuse_sheen_terms + torch.lerp( diffuse_lambertian_terms, diffuse_subsurface_terms, torch.unsqueeze(subsurface, dim=3)) combined_terms = (1 - torch.unsqueeze(metallic, dim=3) ) * diffuse_terms + specular_terms + torch.unsqueeze( clearcoat_amounts, dim=3) * 0.25 * clearcoat_terms return visible * combined_terms