def _generate_SVBRDF_directions(
num_rows: int, num_cols: int) -> Tuple[Tensor, Tensor, Tensor]:
'''
Generates a set of normals and incident-outbound direction pairs which match the θ[h] and θ[d] parameterization of the
MERL 100 BRDF slices.
Args:
num_rows: Number of θ[d] samples in the BRDF slice.
num_cols: Number of θ[h] samples in the BRDF slice.
Returns:
Tensor [R, C, 1, 3] of normals, incident directions, and outbound directions.
'''
normals = torch.tensor([0, 0, 1],
dtype=torch.float64).expand(num_rows, num_cols, 3)
# An incident direction corresponding to a (θ[h], θ[d]) pair is initialized by moving from the normal along the
# positive X-axis until its zenith angle is θ[d]. The incident direction is then rotated along the positive Y-axis
# by θ[h] degrees.
angles = utils.create_grid(num_rows,
num_cols)[:, :, :2].double() * (math.pi / 2)
half_angles = angles[:, :, 0]
diff_angles = angles[:, :, 1].flip(0)
incident_directions = torch.stack([
torch.sin(diff_angles),
torch.sin(half_angles) * torch.cos(diff_angles),
torch.cos(half_angles) * torch.cos(diff_angles)
],
dim=2)
# The outbound direction corresponding to an incident direction is found by flipping its X-coordinate.
outbound_directions = incident_directions * torch.tensor([-1, 1, 1])
return normals.unsqueeze(2), incident_directions.unsqueeze(
2), outbound_directions.unsqueeze(2)
def _shade_render_save(normals: Tensor, svbrdf: SVBRDF, lights: List[Light],
viewer: Viewer, camera: Camera, path: str) -> None:
'''
Shades, renders, and saves a picture to the given path from the provided normals, Lights, Viewer, Camera, and SVBRDF.
Args:
normals: Tensor [1, R, C, 3] of normals to use for shading.
svbrdf: SVBRDF to use for shading.
lights: Lights to use for shading.
viewer: Viewer to use for shading.
camera: Camera to use for rendering.
path: Path to use for saving the image.
'''
num_rows, num_cols = normals.size(1), normals.size(2)
num_size = max(num_rows, num_cols)
surface = (utils.create_grid(num_rows=num_rows, num_cols=num_cols) -
torch.tensor([0.5, 0.5, 0])) * torch.tensor(
[num_cols / num_size, num_rows / num_size, 1])
radiance = shader.shade(surface=surface,
normals=normals,
lights=lights,
viewer=viewer,
svbrdf=svbrdf)
picture = camera.render(surface=surface, radiance=radiance[0])
image.save(path=path, image=picture, encoding='sRGB')
def individual_to_image(individual, image_size=settings.image_size):
if settings.show_generation_time:
print("--------------------------------------")
print("Generating image...")
start = timer()
inputs = utils.create_grid(image_size, image_size, individual.scale)
inputs = inputs[:settings.nb_input_params] # + (individual.latent_vector,)
inputs = np.concatenate(inputs, axis=1)
with torch.no_grad():
inputs = torch.Tensor(inputs).to(settings.device)
output = individual.net(inputs)
image = []
for channel in range(settings.channels):
chan_data = output[:, channel].cpu().numpy()
chan_data = utils.normalize(chan_data, 255)
image.append(chan_data.reshape(image_size, image_size))
image = np.dstack(image)
if settings.channels == 1:
image = np.repeat(image, 3, axis=2)
if settings.show_generation_time:
end = timer()
print("Generation took : ", end - start, " seconds.")
print("--------------------------------------")
return image
def main(year, dx=100000, snowType='SRLD', extraStr='v11'):
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
print(xptsG)
print(yptsG)
dxStr = str(int(dx / 1000)) + 'km'
print(dxStr)
#region_mask, xptsI, yptsI = cF.get_region_mask_pyproj(anc_data_path, proj, xypts_return=1)
#region_maskG = griddata((xptsI.flatten(), yptsI.flatten()), region_mask.flatten(), (xptsG, yptsG), method='nearest')
#xptsDays, yptsDays, oibdates, snowDays= cF.read_icebridge_snowdepths(proj, oib_data_path, year)
xptsDays, yptsDays, _, _, snowDays, oibdates = cF.getSTOSIWIGyear_proj(
proj, oib_data_path + '/stosiwig/', snowType, year)
for x in range(len(oibdates)):
# Loop through dates of each flight. I want to keep separate to compre to the daily NESOSIM data.
oib_dayG = bin_oib(dx, xptsDays[x], yptsDays[x], xptsG, yptsG,
snowDays[x])
cF.plot_gridded_cartopy(
lonG,
latG,
oib_dayG,
proj=ccrs.NorthPolarStereo(central_longitude=-45),
out=figure_path + '/OIB/' + oibdates[x] + dxStr + snowType +
extraStr,
date_string=oibdates[x],
month_string='',
varStr='OIB snow depth ',
units_lab=r'm',
minval=0,
maxval=0.6,
cmap_1=plt.cm.viridis)
oib_dayG.dump(forcing_save_path + dxStr + '/OIB/' + str(year) + '/' +
oibdates[x] + dxStr + snowType + extraStr)
arr = np.hstack(xptsDays)
oib_daysG = cF.bin_oib(dx, np.hstack(xptsDays), np.hstack(yptsDays), xptsG,
yptsG, np.hstack(snowDays))
cF.plot_gridded_cartopy(lonG,
latG,
oib_daysG,
proj=ccrs.NorthPolarStereo(central_longitude=-45),
out=figure_path + '/OIB/' + str(year) + dxStr +
snowType + extraStr,
date_string=str(year),
month_string='',
varStr='OIB snow depth ',
units_lab=r'm',
minval=0,
maxval=0.6,
cmap_1=plt.cm.viridis)
def main(year, dx=100000, extraStr='v11'):
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
#print(xptsG)
#print(yptsG)
dxStr = str(int(dx / 1000)) + 'km'
#print(dxStr)
files = glob(forcing_save_path + dxStr + '/OIB/' + str(year) + '/' + '*' +
dxStr + extraStr + '')
print(files)
oibdates = [file[-16:-8] for file in files]
print(oibdates)
for x in range(len(oibdates)):
# Loop through dates of each flight. I want to keep separate to compre to the daily NESOSIM data.
try:
oib_dayG1 = np.load(forcing_save_path + dxStr + '/OIB/' +
str(year) + '/' + oibdates[x] + dxStr +
'GSFC' + extraStr,
allow_pickle=True)
oib_dayG2 = np.load(forcing_save_path + dxStr + '/OIB/' +
str(year) + '/' + oibdates[x] + dxStr +
'SRLD' + extraStr,
allow_pickle=True)
oib_dayG3 = np.load(forcing_save_path + dxStr + '/OIB/' +
str(year) + '/' + oibdates[x] + dxStr + 'JPL' +
extraStr,
allow_pickle=True)
oib_dayGC = np.median((oib_dayG1, oib_dayG2, oib_dayG3), axis=0)
cF.plot_gridded_cartopy(
lonG,
latG,
oib_dayGC,
proj=ccrs.NorthPolarStereo(central_longitude=-45),
out=figure_path + '/OIB/' + oibdates[x] + dxStr + 'MEDIAN' +
extraStr,
date_string=str(year),
month_string='',
varStr='OIB snow depth MEDIAN',
units_lab=r'm',
minval=0,
maxval=0.6,
cmap_1=plt.cm.viridis)
oib_dayGC.dump(forcing_save_path + dxStr + '/OIB/' + str(year) +
'/' + oibdates[x] + dxStr + 'MEDIAN' + extraStr)
except:
print('All thre algs dont exist for this date:', oibdates[x])
def render(self, surface: Tensor, radiance: Tensor) -> Tensor:
'''See Camera.render().'''
# Determine the height and width of a surface texel.
surface_row_step = (surface[-1, 0] - surface[0, 0]) / (surface.size(0) - 1)
surface_col_step = (surface[0, -1] - surface[0, 0]) / (surface.size(1) - 1)
# Compute the origin of the surface taking into account that each point represents the center of a texel.
surface_origin = surface[0, 0] - surface_row_step / 2 - surface_col_step / 2
# Derive the axes of the surface which are aligned with the row and column structure of the points.
surface_row_axis = surface_row_step * surface.size(0)
surface_col_axis = surface_col_step * surface.size(1)
tiled_surface_row_axis = surface_row_axis.expand(int(self.resolution[1]), int(self.resolution[0]), 3)
tiled_surface_col_axis = surface_col_axis.expand(int(self.resolution[1]), int(self.resolution[0]), 3)
# Construct an image plane coincident with the Cartesian plane from the focal position (0, 0, 1).
image_plane = create_grid(int(self.resolution[1]), int(self.resolution[0])) * 2 - 1
image_plane[:, :, 0] *= torch.tan(math.pi * self.field_of_view[0] / 360)
image_plane[:, :, 1] *= torch.tan(math.pi * self.field_of_view[1] / 360)
# Derive an appropriate basis for the image plane based on the direction of the camera.
image_plane_basis = create_orthonormal_basis(self.direction)
# Transform the image plane into the reference frame of the camera.
camera_ray_directions = image_plane_basis[2] * image_plane[:, :, [0]] - image_plane[:, :, [1]] * image_plane_basis[1] + image_plane_basis[0]
# The intersection between a camera ray and the surface is given by the Möller–Trumbore algorithm.
collision_mats = torch.stack([tiled_surface_row_axis, tiled_surface_col_axis, -camera_ray_directions], dim=2)
collision_mats = torch.unsqueeze(collision_mats, dim=2).repeat(1, 1, 4, 1, 1).transpose(-1, -2)
surface_to_camera_ray = self.position - surface_origin
# The algorithm boils down to solving the following system of equations with Cramer's rule:
# [ surface_row_axis.x surface_col_axis.x -camera_ray_directions.x ] [ A ] [ surface_to_camera_ray.x ]
# | surface_row_axis.y surface_col_axis.y -camera_ray_directions.y | x | B | = | surface_to_camera_ray.y |
# [ surface_row_axis.z surface_col_axis.z -camera_ray_directions.z ] [ C ] [ surface_to_camera_ray.z ]
collision_mats[:, :, 0, :, 0] = surface_to_camera_ray
collision_mats[:, :, 1, :, 1] = surface_to_camera_ray
collision_mats[:, :, 2, :, 2] = surface_to_camera_ray
collision_dets = torch.det(collision_mats)
# Solve the system of equations to find the collision scalars.
collision_vars = collision_dets[:, :, :3] / collision_dets[:, :, [-1]].expand(-1, -1, 3)
# Discard any collisions that occurred behind the camera.
collision_vars[:, :, :2] *= collision_vars[:, :, [2]].sign().clamp(0, 1).expand(-1, -1, 2)
# Sample the radiosity from the collision coordinates using bilinear filtering.
image = self.exposure * torch.nn.functional.grid_sample(input=torch.unsqueeze(radiance.permute(2, 0, 1), dim=0),
grid=torch.unsqueeze(collision_vars[:, :, [1, 0]], dim=0) * 2 - 1,
mode='bilinear',
padding_mode='zeros',
align_corners=False).squeeze().permute(1, 2, 0)
logging.info('Rendered %dx%d image of a %dx%d surface', image.size(1), image.size(0), surface.size(1), surface.size(0))
return image
def _feedback_flow(config: Configuration) -> None:
'''
The "feedback" flow iteratively infers the SVBRDF parameters of a texture, renders it, and feeds the output of the
rendering back into the network. The purpose of this flow is to test the robustness of an SVBRDF autoencoder.
Args:
config: Configuration specifying the parameters of the flow.
'''
with torch.no_grad():
autoencoder, svbrdf, camera, (feedback_lights, feedback_viewer), (
rendering_lights, rendering_viewer
), input_path, output_path, loops = config.load_feedback_flow()
autoencoder.eval()
# It is assumed that the dimensions of the input image will be accepted by the network.
input_image = image.load(path=input_path, encoding='sRGB')
num_texture_rows = input_image.size(0)
num_texture_cols = input_image.size(1)
input_distance = utils.create_radial_distance_field(
num_rows=num_texture_rows, num_cols=num_texture_cols)
# By convention, PyTorch expects Tensors to be in [B, D, R, C] format.
input_batch = torch.cat([input_image, input_distance],
dim=2).unsqueeze(0).permute(0, 3, 1, 2)
normals, svbrdf.parameters = SVBRDFAutoencoder.interpret(
autoencoder.forward(input_batch))
surface = utils.create_grid(num_rows=num_texture_rows,
num_cols=num_texture_cols)
for i in tqdm.tqdm(range(loops), desc='Feedback Looping'):
# The slightly-awkward ordering of statements before and inside the loops ensures that |loops| can be set to zero.
input_image = shader.shade(surface=surface,
normals=normals,
lights=feedback_lights,
viewer=feedback_viewer,
svbrdf=svbrdf)[0]
input_batch = torch.cat([input_image, input_distance],
dim=2).unsqueeze(0).permute(0, 3, 1, 2)
normals, svbrdf.parameters = SVBRDFAutoencoder.interpret(
autoencoder.forward(input_batch))
_shade_render_save(normals=normals,
svbrdf=svbrdf,
lights=rendering_lights,
viewer=rendering_viewer,
camera=camera,
path=output_path)
def __init__(self, dims: Dict, path: str, layout: Dict,
textures: List[Texture], transforms: List[Transform],
svbrdf: SVBRDF, lights: List[Light], viewer: Viewer) -> None:
'''
Constructs a new Dataset with the given dimensions, path, layout, textures, SVBRDF, Lights, and Viewer.
Args:
dims: Dimensions of the Dataset textures and crops.
path: Path to the root directory of the Dataset.
layout: Filesystem layout of each element in the Dataset.
textures: Texture descriptions which comprise the Dataset.
transforms: Transforms to be applied to a sample from the Dataset.
svbrdf: SVBRDF associated with the parameter maps in the Dataset.
lights: Lights used to shade a texture.
viewer: Viewer used to shade a texture.
'''
for key in ('Texture', 'Crop'):
assert key in dims, f'Dimensions dictionary is missing key "{key}".'
for key in ('Normals', 'Parameters'):
assert key in layout, f'Layout dictionary is missing key "{key}".'
for i, parameter in enumerate(layout['Parameters']):
assert 'Type' in parameter, f'Parameter {i} is missing key "Type" in layout dictionary.'
assert 'Name' in parameter, f'Parameter {i} is missing key "Name" in layout dictionary.'
self._dims = dims
self._path = path
self._layout = layout
self._textures = textures
self._transforms = transforms
self._svbrdf = svbrdf
self._lights = lights
self._viewer = viewer
# A rendering surface is needed to generate flash-lit images for consumption by an SVBRDF autoencoder network.
# Similarly, a radial distance field (indicating the distance from each point on the surface to the center of
# the surface) enables the network to discriminate between flash-lit and non-flash-lit regions.
num_crop_rows = self._dims['Crop'][0]
num_crop_cols = self._dims['Crop'][1]
self._surface = utils.create_grid(num_rows=num_crop_rows,
num_cols=num_crop_cols)
self._radial_distance_field = utils.create_radial_distance_field(
num_rows=num_crop_rows, num_cols=num_crop_cols).unsqueeze(0)
def compute_radiance(network_normals: Tensor, network_svbrdf: SVBRDF,
dataset_normals: Tensor,
dataset_svbrdf: SVBRDF) -> Tuple[Tensor, Tensor]:
'''
Computes the radiance from the given normals and SVBRDFs with respect to a random point Light and Viewer.
Args:
network_normals: Tensor [B, R, C, 3] of SVBRDF autoencoder normals.
network_svbrdf: SVBRDF with embedded SVBRDF autoencoder parameters.
dataset_normals: Tensor [B, R, C, 3] of ground-truth normals.
dataset_svbrdf: SVBRDF with embedded ground-truth parameters.
Returns:
Tuple containing the SVBRDF autoencoder and Dataset radiance Tensors.
'''
# There is no harm in sharing the same drawing canvas for both the network and dataset renderings.
texture_rows = dataset_normals.size(1)
texture_cols = dataset_normals.size(2)
surface = utils.create_grid(num_rows=texture_rows, num_cols=texture_cols)
# The Light and Viewer are sampled from a cosine-weighted distribution following the Single-Image SVBRDF Capture
# with a Rendering-Aware Deep Network paper.
origin = torch.tensor([0.5, 0.5, 0.0], device=utils.get_device_name())
lights = [
PunctualLight(position=utils.sample_cosine_hemisphere(origin),
lumens=torch.rand(1).expand(3) * 2 + 0.5)
]
viewer = PerspectiveViewer(position=utils.sample_cosine_hemisphere(origin))
network_radiance = shader.shade(surface=surface,
normals=network_normals,
lights=lights,
viewer=viewer,
svbrdf=network_svbrdf)
dataset_radiance = shader.shade(surface=surface,
normals=dataset_normals,
lights=lights,
viewer=viewer,
svbrdf=dataset_svbrdf)
return network_radiance, dataset_radiance
saver.restore(sess, tf.train.latest_checkpoint(model_path))
graph = tf.get_default_graph()
x = graph.get_tensor_by_name("x:0")
if len(data) % batch == 0:
for i in range(len(data) // batch):
minibatch = data[i * batch:(i + 1) * batch]
whoreco_result(minibatch, i * batch)
elif len(data) < batch:
minibatch = data[0:len(data)]
partreco_result(minibatch, 0)
else:
whoRange = len(data) // batch
partRange = len(data) % batch
for i in range(whoRange):
minibatch = data[i * batch:(i + 1) * batch]
whoreco_result(minibatch, i * batch)
minibatch = data[whoRange * batch:len(data)]
partreco_result(minibatch, whoRange * batch)
result_output, crsave = selectResult(result_output, crsave)
os.remove(ab_rootPath)
filetime, generateTime, title = utils.create_grid(
crsave, result_output, path, gridH, gridC, station)
utils.create_bin(crsave, result_output, title, filetime,
generateTime, station, gridH, gridC,
radarCount, startLon, startLat, endLon,
endLat, XReso, YReso)
except:
os.remove(ab_rootPath)
pass
record_count += 1
print(record_count)
opt.zero_grad()
masked_img = batch["masked_image"].to(device).float()
mask = batch["mask"].to(device).float()
image = batch["image"].to(device)
pred = NET(masked_img, mask)
loss.prepare_loss_calculation(pred, image, mask)
loss_hole = loss.calculate_loss_hole()
loss_valid = loss.calculate_loss_valid()
perceptual_loss = loss.calculate_perceptual_loss()
style_loss_out = loss.calculate_style_out_loss()
style_loss_comp = loss.calculate_style_comp_loss()
tv_loss = loss.calculate_tv_loss()
actual_loss = loss_valid + 6*loss_hole + 0.05*perceptual_loss + \
120*(style_loss_out + style_loss_comp) + 0.1*tv_loss
if GLOBAL_STEP % 3000 == 0:
print(actual_loss)
grid = create_grid(masked_img, pred)
write_to_tensorboard(writer, grid, actual_loss, GLOBAL_STEP)
write_to_tensorboard(writer, None, actual_loss, GLOBAL_STEP)
actual_loss.backward()
opt.step()
torch.cuda.empty_cache()
GLOBAL_STEP += 1
model_path = os.path.join(checkpoint_path, str(epoch))
torch.save(NET.state_dict(), model_path)
def main(yearT,
extraStr='v11_1',
dx=100000,
data_path=reanalysis_raw_path + 'ERA5/',
out_path=forcing_save_path + 'Temp/ERA5/',
fig_path=figure_path + 'Temp/ERA5/',
anc_data_path='../../anc_data/'):
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
print(xptsG)
print(yptsG)
dxStr = str(int(dx / 1000)) + 'km'
print(dxStr)
if not os.path.exists(out_path):
os.makedirs(out_path)
if not os.path.exists(fig_path):
os.makedirs(fig_path)
region_mask, xptsI, yptsI, _, _ = cF.get_region_mask_pyproj(anc_data_path,
proj,
xypts_return=1)
region_maskG = griddata((xptsI.flatten(), yptsI.flatten()),
region_mask.flatten(), (xptsG, yptsG),
method='nearest')
varStr = 't2m'
xptsM, yptsM, temp2mYear = get_ERA5_temps(proj, data_path, yearT)
hotdays_duration = xr.apply_ufunc(get_cumulative_hotdays,
temp2mYear['t2m'].compute(),
input_core_dims=[["time"]],
vectorize=True)
print(xptsM.flatten().shape)
print(yptsM.flatten().shape)
print(temp2mYear)
print(hotdays_duration.values.flatten().shape)
t2mdurG = griddata((xptsM.flatten(), yptsM.flatten()),
hotdays_duration.values.flatten(), (xptsG, yptsG),
method='linear')
#t2mdurG[where(region_maskG>10)]=0.
#t2mdur=t2mdur.astype('f2')
cF.plot_gridded_cartopy(lonG,
latG,
t2mdurG,
proj=ccrs.NorthPolarStereo(central_longitude=-45),
out=fig_path + '/duration' + str(yearT) + extraStr,
date_string=str(yearT),
extra=extraStr,
varStr='hot days',
units_lab=r'>0',
minval=0,
maxval=100,
cmap_1=plt.cm.viridis)
#monthStr='%02d' %(month+1)
t2mdurG.dump(out_path + '/ERA5_duration' + dxStr + '-' + str(yearT) +
extraStr)
def main(year,
startMonth=0,
endMonth=11,
extraStr='v11',
dx=100000,
data_path=nsidc_raw_path,
out_path=forcing_save_path,
fig_path=figure_path + 'IceDrift/NSIDCv4/',
anc_data_path='../../AncData/'):
print(data_path)
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
# create source and target cartopy projections
srcProj = cF.EASE_North()
newProj = cF.P3413()
print(xptsG)
print(yptsG)
dxStr = str(int(dx / 1000)) + 'km'
print(dxStr)
numDays = cF.getLeapYr(year)
if (numDays > 365):
monIndex = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
else:
monIndex = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
if not os.path.exists(out_path + '/' + dxStr + '/IceDrift/NSIDCv4/' +
str(year)):
os.makedirs(out_path + '/' + dxStr + '/IceDrift/NSIDCv4/' + str(year))
if not os.path.exists(fig_path):
os.makedirs(fig_path)
for month in range(startMonth, endMonth + 1):
print(month)
numDays = monIndex[month]
for x in range(numDays):
dayT = sum(monIndex[0:month]) + x
dayStr = '%03d' % dayT
print(dayStr)
print('Drift, year:', year, 'day:', dayStr)
xeasedrift, yeasedrift, lont, latt = cF.get_nsidc_driftv4(
data_path, year, dayT, 'daily')
u_rot, v_rot = newProj.transform_vectors(srcProj, lont, latt,
xeasedrift, yeasedrift)
drift_day_xy = np.stack((u_rot, v_rot))
xpts, ypts = proj(lont, latt)
drift_xyG = cF.int_smooth_drifts_v2(xptsG,
yptsG,
xpts,
ypts,
latt,
drift_day_xy,
sigma_factor=0.5)
#drift_day_xy[1] = vy
print(drift_xyG)
cF.plot_drift_cartopy(lonG,
latG,
xptsG,
yptsG,
drift_xyG[0],
drift_xyG[1],
np.sqrt(drift_xyG[0]**2 + drift_xyG[1]**2),
out=fig_path + str(year) + '_d' + dayStr +
dxStr + extraStr,
units_lab='m/s',
units_vec=r'm s$^{-1}$',
minval=0,
maxval=0.5,
vector_val=0.1,
date_string=str(year) + '_d' + dayStr,
month_string='',
varStr='NSIDCv4 ice drift ',
cbar_type='max',
cmap_1=plt.cm.viridis)
drift_xyG.dump(out_path + '/' + dxStr + '/IceDrift/NSIDCv4/' +
str(year) + '/NSIDCv4_driftG' + dxStr + '-' +
str(year) + '_d' + dayStr + extraStr)
def main(year,
startMonth=8,
endMonth=11,
dx=100000,
extraStr='v11_1',
data_path=reanalysis_raw_path + 'ERA5/',
out_path=forcing_save_path + 'Precip/ERA5/',
fig_path=figure_path + 'Precip/ERA5/',
anc_data_path='../../anc_data/'):
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
print(xptsG)
print(yptsG)
dxStr = str(int(dx / 1000)) + 'km'
print(dxStr)
region_mask, xptsI, yptsI, _, _ = cF.get_region_mask_pyproj(anc_data_path,
proj,
xypts_return=1)
region_maskG = griddata((xptsI.flatten(), yptsI.flatten()),
region_mask.flatten(), (xptsG, yptsG),
method='nearest')
varStr = 'sf'
if not os.path.exists(fig_path):
os.makedirs(fig_path)
yearT = year
numDays = cF.getLeapYr(year)
if (numDays > 365):
monIndex = [0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366]
else:
monIndex = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365]
if not os.path.exists(out_path + '/' + str(year)):
os.makedirs(out_path + '/' + str(year))
startDay = monIndex[startMonth]
if (endMonth > 11):
endDay = monIndex[endMonth + 1 - 12] + monIndex[-1] - 1
else:
endDay = monIndex[endMonth + 1]
calc_weights = 1 # start as one to calculate weightings then gets set as zero for future files
for dayT in range(startDay, endDay):
dayStr = '%03d' % dayT
month = np.where(dayT - np.array(monIndex) >= 0)[0][-1]
monStr = '%02d' % (month + 1)
dayinmonth = dayT - monIndex[month]
print('Precip day:', dayT, dayinmonth)
#in kg/m2 per day
xptsM, yptsM, lonsM, latsM, Precip = cF.get_ERA5_precip_days_pyproj(
proj,
data_path,
str(yearT),
monStr,
dayinmonth,
lowerlatlim=30,
varStr=varStr)
# if it's the first day, calculate weights
if calc_weights == 1:
# calculate Delaunay triangulation interpolation weightings for first file of the year
print('calculating interpolation weightings')
ptM_arr = np.array([xptsM.flatten(), yptsM.flatten()]).T
tri = Delaunay(ptM_arr) # delaunay triangulation
calc_weights = 0
# grid using linearNDInterpolator with triangulation calculated above
# (faster than griddata but produces identical output)
interp = LinearNDInterpolator(tri, Precip.flatten())
PrecipG = interp((xptsG, yptsG))
cF.plot_gridded_cartopy(
lonG,
latG,
PrecipG,
proj=ccrs.NorthPolarStereo(central_longitude=-45),
out=fig_path + '/ERA5' + varStr + dxStr + '-' + str(yearT) + '_d' +
dayStr + extraStr,
date_string=str(yearT),
month_string=str(dayT),
extra=extraStr,
varStr='ERA5 snowfall ',
units_lab=r'kg/m2',
minval=0,
maxval=10,
cmap_1=plt.cm.viridis)
PrecipG.dump(out_path + str(yearT) + '/ERA5' + varStr + dxStr + '-' +
str(yearT) + '_d' + dayStr + extraStr)
def main(yearT,
extraStr='v11_1',
dx=100000,
data_path=reanalysis_raw_path + 'ERA5/',
out_path=forcing_save_path + 'InitialConditions/ERA5/',
fig_path=figure_path + 'InitialConditions/ERA5/',
anc_data_path='../../anc_data/'):
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
print(xptsG)
print(yptsG)
dxStr = str(int(dx / 1000)) + 'km'
print(dxStr)
if not os.path.exists(out_path):
os.makedirs(out_path)
if not os.path.exists(fig_path):
os.makedirs(fig_path)
region_mask, xptsI, yptsI, _, _ = cF.get_region_mask_pyproj(anc_data_path,
proj,
xypts_return=1)
region_maskG = griddata((xptsI.flatten(), yptsI.flatten()),
region_mask.flatten(), (xptsG, yptsG),
method='nearest')
reanalysis = 'ERA5'
varStr = 't2m'
iceconc_path = forcing_save_path + '/IceConc/CDR/'
temp_path = forcing_save_path + '/Temp/ERA5/'
t2mdurGAll = []
for y in range(1980, 1991 + 1, 1):
if (y == 1987):
continue
t2mdurGT = np.load(temp_path + 'duration' + dxStr + '-' + str(y) +
extraStr,
allow_pickle=True)
t2mdurGAll.append(t2mdurGT)
t2mdurGclim = ma.mean(t2mdurGAll, axis=0)
print(t2mdurGclim.shape)
w99 = cF.getWarren(lonG, latG, 7)
for yearT in range(2021, 2021 + 1, 1):
if (yearT == 1987):
continue
t2mdurGT = np.load(temp_path + 'duration' + dxStr + '-' + str(yearT) +
extraStr,
allow_pickle=True)
print(t2mdurGT.shape)
W99yrT = w99 * (t2mdurGclim / t2mdurGT)
W99yrT[np.where(latG < 70)] = 0
W99yrT[np.where(region_maskG > 8.2)] = 0
W99yrT[np.where(region_maskG <= 7.8)] = 0
W99yrT[np.where(W99yrT < 0)] = 0
W99yrT[np.where(W99yrT > 10)] = 10
day = 226
dayStr = str(day) #226 is middle of August
iceConcDayG = np.load(iceconc_path + str(yearT) + '/iceConcG_CDR' +
dxStr + '-' + str(yearT) + '_d' + dayStr +
extraStr,
allow_pickle=True)
W99yrT[np.where(iceConcDayG < 0.15)] = 0
W99yrT = gaussian_filter(W99yrT, sigma=1)
# Convert to meters
W99yrT = W99yrT / 100.
cF.plot_gridded_cartopy(
lonG,
latG,
W99yrT,
proj=ccrs.NorthPolarStereo(central_longitude=-45),
out=fig_path + '/initial_conditions' + str(yearT) + dxStr +
extraStr,
date_string=str(yearT),
extra=extraStr,
varStr='Snow depth ',
units_lab=r'm',
minval=0,
maxval=0.12,
cmap_1=plt.cm.viridis)
W99yrT.dump(out_path + 'ICsnow' + dxStr + '-' + str(yearT) + extraStr)
from valueIteration import do_several_value_iterations from policyIteration import do_several_policy_iterations from utils import create_grid file_name = 'i1.txt' # create a new Grid rlt = create_grid(file_name) # [size, discount_factor, noises, states] grid_size = rlt[0] discount_factor = rlt[1] noises = rlt[2] states = rlt[3] do_several_value_iterations(discount_factor, noises, states, grid_size) rlt = create_grid(file_name) # [size, discount_factor, noises, states] grid_size = rlt[0] discount_factor = rlt[1] noises = rlt[2] states = rlt[3] do_several_policy_iterations(discount_factor, noises, states, grid_size)
imsize = (256, 256)
center = (128, 128)
gauss_peak = 0
dist_from_core = 24
gauss_bmaj = 10
gauss_e = 1.
gauss_bpa = 0.
gauss_peak_jet = 0.0
dist_from_core_jet = 24
gauss_bmaj_jet = 10
gauss_e_jet = 1.
gauss_bpa_jet = 0.
cut = 0.000001
transverse = 'gauss'
x, y = create_grid(imsize)
x -= center[0]
y -= center[1]
max_flux = float(max_flux)
along = np.where(x > 0, 0, 0)
if transverse == 'linear':
perp = -(2 * max_flux / width) * abs(y)
elif transverse == 'quadratic':
perp = -(max_flux / (width / 2)**2.) * y**2.
elif transverse == 'sqrt':
perp = -(max_flux / np.sqrt(width / 2.)) * np.sqrt(abs(y))
elif transverse == 'gauss':
gauss_width = 0.3 * x
perp = np.where(x > 0.1,
max_flux * np.exp(-y**2 / (2. * gauss_width**2)) / x, 0)
train_loader, test_loader, N_train, N_test, z = data.train_test_split_latents(net,
experiment_parameters,
x_train,
x_test,
batch_size=batch_size)
if experiment_parameters["load_land"]:
mu = torch.Tensor(np.load(join_path(model_dir, 'land_mu.npy'))).to(device).requires_grad_(True)
A = torch.Tensor(np.load(join_path(model_dir, 'land_std.npy'))).to(device).requires_grad_(True)
else:
mu = stats.sturm_mean(net, z.to(device), num_steps=5).unsqueeze(0)
# meshgrid creating
meshsize = 100 if experiment_parameters["sampled"] else 20
Mxy, dv = utils.create_grid(z, meshsize)
# curves = {} # for init curves, but they seem useless for now
if experiment_parameters["sampled"]:
with torch.no_grad():
grid_prob, grid_metric, grid_metric_sum, grid_save = land.LAND_grid_prob(grid=Mxy,
model=net,
batch_size=1,
device=device)
else:
grid_metric_sum = None
if not experiment_parameters["load_land"]:
mus, average_loglik, stds = [], [], []
for i in range(10):
def main(year, startMonth=3, endMonth=3, extraStr='v11_1', dx=100000, data_path=cdr_raw_path, out_path=forcing_save_path, fig_path=figure_path+'IceConc/CDR/', anc_data_path='../../anc_data/'):
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
print(xptsG)
print(yptsG)
dxStr=str(int(dx/1000))+'km'
print(dxStr)
region_mask, xptsI, yptsI, lonsI, latsI = cF.get_region_mask_pyproj(anc_data_path, proj, xypts_return=1)
region_maskG = griddata((xptsI.flatten(), yptsI.flatten()), region_mask.flatten(), (xptsG, yptsG), method='nearest')
product='CDR'
numDaysYr=cF.getLeapYr(year)
if (numDaysYr>365):
monIndex = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
else:
monIndex = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
if not os.path.exists(out_path+'/'+str(year)):
os.makedirs(out_path+'/'+str(year))
if not os.path.exists(fig_path):
os.makedirs(fig_path)
calc_weights = 1 # start as one to calculate weightings then gets set as zero for future files
for month in range(startMonth, endMonth+1):
print(month)
mstr='%02d' %(month+1)
# Get pole hole
pmask=cF.get_pmask(year, month)
numDays=monIndex[month]
# should return array with nans not masked as needed for regridding.
for x in range(numDays):
dayT=sum(monIndex[0:month])+x
daySumStr='%03d' %(dayT)
dayMonStr='%02d' %(x+1)
print('day month string', dayMonStr)
try:
# Try final first
fileT=glob(data_path+'/final/'+str(year)+'/'+mstr+'/*'+str(year)+mstr+dayMonStr+'*.nc')[0]
print(data_path+'/final/'+str(year)+'/'+mstr+'/*'+str(year)+mstr+dayMonStr+'*.nc')
print('final data')
except:
try:
# Try nrt
fileT=glob(data_path+'/nrt/'+str(year)+'/'+mstr+'/*'+str(year)+mstr+dayMonStr+'*.nc')[0]
print(data_path+'/nrt/'+str(year)+'/'+mstr+'/*'+str(year)+mstr+dayMonStr+'*.nc')
print('nrt data')
except:
try:
dayMonStr='%03d' %(dayT-1)
fileT=glob(data_path+'/final/'+str(year)+'/'+mstr+'/*'+str(year)+mstr+dayMonStr+'*.nc')[0]
print(data_path+'/final/'+str(year)+'/'+mstr+'/*'+str(year)+mstr+dayMonStr+'*.nc')
print('previous day final data')
except:
try:
dayMonStr='%03d' %(dayT+1)
fileT=glob(data_path+'/final/'+str(year)+'/'+mstr+'/*'+str(year)+mstr+dayMonStr+'*.nc')[0]
print(data_path+'/final/'+str(year)+'/'+mstr+'/*'+str(year)+mstr+dayMonStr+'*.nc')
print('following day final data')
except:
print('no conc')
# previously was pass but this meant it would just use the previous loop data below
continue
iceConcDay = cF.getCDRconcproj(fileT, mask=0, maxConc=1, lowerConc=1)
print(iceConcDay.shape)
# if it's the first day, calculate weights
if calc_weights == 1:
# calculate Delaunay triangulation interpolation weightings for first file of the year
print('calculating interpolation weightings')
# Use region mask which is on the same grid!
ptM_arr = np.array([xptsI.flatten(),yptsI.flatten()]).T
tri = Delaunay(ptM_arr) # delaunay triangulation
calc_weights = 0
iceConcDay[np.where(region_mask>10)]=np.nan
#iceConcDayG = griddata((xpts0.flatten(), ypts0.flatten()), iceConcDay.flatten(), (xptsG, yptsG), method='linear')
# grid using linearNDInterpolator with triangulation calculated above
# (faster than griddata but produces identical output)
interp = LinearNDInterpolator(tri,iceConcDay.flatten())
iceConcDayG = interp((xptsG,yptsG))
iceConcDayG[np.where(iceConcDayG<0.15)]=0
iceConcDayG[np.where(iceConcDayG>1)]=1
iceConcDayG[np.where(region_maskG>10)]=np.nan
cF.plot_gridded_cartopy(lonG, latG, iceConcDayG, proj=ccrs.NorthPolarStereo(central_longitude=-45), out=fig_path+'iceConcG_-'+str(year)+mstr+dayMonStr+dxStr+extraStr,
date_string=str(year), month_string=mstr+dayMonStr, extra=extraStr, varStr='CDR ice conc', units_lab='', minval=0, maxval=1, cmap_1=plt.cm.viridis)
iceConcDayG.dump(out_path+str(year)+'/iceConcG_CDR'+dxStr+'-'+str(year)+'_d'+daySumStr+extraStr)
import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import interp2d
import sys
sys.path.append('../')
import utils as cF
import cartopy.crs as ccrs
import pandas as pd
import xarray as xr
ancDataPath = '../../anc_data/'
dx = 100000 # adjust to change resolution
dxStr = str(int(dx / 1000)) + 'km'
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
print('grid generated')
print(latG.shape)
lat_len = latG.shape[0]
lat_60_idx = np.argmin(np.abs(latG[:, lat_len // 2] -
60)) # index of lat closest to 60
# check which side of the centre the limit is on
if lat_60_idx < lat_len // 2:
lim_low = lat_60_idx
lim_high = lat_len - lat_60_idx
else:
lim_low = lat_len - lat_60_idx
def main(year1, month1, day1, year2, month2, day2, outPathT='.', forcingPathT='.', anc_data_pathT='../anc_data/', figPathT='../Figures/',
precipVar='ERA5', windVar='ERA5', driftVar='OSISAF', concVar='CDR', icVar='ERAI', densityTypeT='variable',
outStr='', extraStr='', IC=2, windPackFactorT=0.1, windPackThreshT=5., leadLossFactorT=0.1, atmLossFactorT=2.2e-8, dynamicsInc=1, leadlossInc=1,
windpackInc=1, atmlossInc=0, saveData=1, plotBudgets=1, plotdaily=1, saveFolder='', dx=50000,scaleCS=False):
"""
Main model function
Args:
The various model configuration parameters
"""
#------- Create map projection
xptsG, yptsG, latG, lonG, proj = cF.create_grid(dxRes=dx)
nx=xptsG.shape[0]
ny=xptsG.shape[1]
dxStr=str(int(dx/1000))+'km'
print(nx, ny, dxStr)
# Assign some global parameters
global dataPath, forcingPath, outPath, ancDataPath
outPath=outPathT+dxStr+'/'
forcingPath=forcingPathT+dxStr+'/'
ancDataPath=anc_data_pathT
print('OutPath:', outPath)
print('forcingPath:', forcingPath)
print('ancDataPath:', ancDataPath)
# Assign density of the two snow layers
global snowDensityFresh, snowDensityOld, minSnowD, minConc, leadLossFactor, atmLossFactor, windPackThresh, windPackFactor, deltaT
snowDensityFresh=200. # density of fresh snow layer
snowDensityOld=350. # density of old snow layer
minSnowD=0.02 # minimum snow depth for a density estimate
minConc=0.15 # mask budget values with a concentration below this value
deltaT=60.*60.*24. # time interval (seconds in a day)
region_mask, xptsI, yptsI = cF.get_region_mask_pyproj(anc_data_pathT, proj, xypts_return=1)
region_maskG = griddata((xptsI.flatten(), yptsI.flatten()), region_mask.flatten(), (xptsG, yptsG), method='nearest')
leadLossFactor=leadLossFactorT # Snow loss to leads coefficient
windPackThresh=windPackThreshT # Minimum winds needed for wind packing
windPackFactor=windPackFactorT # Fraction of snow packed into old snow layer
atmLossFactor=atmLossFactorT # Snow loss to atmosphere coefficient
#---------- Current year
yearCurrent=year1
#--------- Get time period info
startDay, numDays, numDaysYear1, dateOut= cF.getDays(year1, month1, day1, year2, month2, day2)
print (startDay, numDays, numDaysYear1, dateOut)
# make this into a small function
dates=[]
for x in range(0, numDays):
#print x
date = datetime.datetime(year1, month1+1, day1+1) + datetime.timedelta(x)
#print (int(date.strftime('%Y%m%d')))
dates.append(int(date.strftime('%Y%m%d')))
#print(dates)
CSstr = ''
if scaleCS:
# load scaling factors; assumes scaling factors are in same directory as NESOSIM.py
monthlyScalingFactors = xr.open_dataset('{}scale_coeffs_{}_{}_v2.nc'.format(ancDataPath, precipVar, dxStr))['scale_factors']
CSstr = 'CSscaled'
#------ create output strings and file paths -----------
saveStr= precipVar+CSstr+'sf'+windVar+'winds'+driftVar+'drifts'+concVar+'sic'+'rho'+densityTypeT+'_IC'+str(IC)+'_DYN'+str(dynamicsInc)+'_WP'+str(windpackInc)+'_LL'+str(leadlossInc)+'_AL'+str(atmlossInc)+'_WPF'+str(windPackFactorT)+'_WPT'+str(windPackThreshT)+'_LLF'+str(leadLossFactorT)+'-'+dxStr+extraStr+outStr+'-'+dateOut
saveStrNoDate=precipVar+CSstr+'sf'+windVar+'winds'+driftVar+'drifts'+concVar+'sic'+'rho'+densityTypeT+'_IC'+str(IC)+'_DYN'+str(dynamicsInc)+'_WP'+str(windpackInc)+'_LL'+str(leadlossInc)+'_AL'+str(atmlossInc)+'_WPF'+str(windPackFactorT)+'_WPT'+str(windPackThreshT)+'_LLF'+str(leadLossFactorT)+'-'+dxStr+extraStr+outStr
print ('Saving to:', saveStr)
#'../../DataOutput/'
savePath=outPath+saveFolder+'/'+saveStrNoDate
# Declare empty arrays for compiling budgets
if not os.path.exists(savePath+'/budgets/'):
os.makedirs(savePath+'/budgets/')
if not os.path.exists(savePath+'/final/'):
os.makedirs(savePath+'/final/')
global figpath
figpath=figPathT+'/Diagnostic/'+dxStr+'/'+saveStrNoDate+'/'
if not os.path.exists(figpath):
os.makedirs(figpath)
if not os.path.exists(figpath+'/daily_snow_depths/'):
os.makedirs(figpath+'/daily_snow_depths/')
precipDays, iceConcDays, windDays, tempDays, snowDepths, density, snowDiv, snowAdv, snowAcc, snowOcean, snowWindPack, snowWindPackLoss, snowWindPackGain, snowLead, snowAtm = genEmptyArrays(numDays, nx, ny)
print('IC:', IC)
if (IC>0):
if (IC==1):
# August Warren climatology snow depths
ICSnowDepth = np.load(forcingPath+'InitialConditions/AugSnow'+dxStr, allow_pickle=True)
print('Initialize with August Warren climatology')
elif (IC==2):
# Petty initiail conditions
try:
ICSnowDepth = np.load(forcingPath+'InitialConditions/'+icVar+'/ICsnow'+dxStr+'-'+str(year1)+extraStr, allow_pickle=True)
print('Initialize with new v1.1 scaled initial conditions')
print(np.amax(ICSnowDepth))
except:
print('No initial conditions file available')
iceConcDayG, precipDayG, driftGdayG, windDayG, tempDayG =loadData(year1, startDay, precipVar, windVar, concVar, driftVar, dxStr, extraStr)
ICSnowDepth[np.where(iceConcDayG<minConc)]=0
#--------Split the initial snow depth over both layers
snowDepths[0, 0]=ICSnowDepth*0.5
snowDepths[0, 1]=ICSnowDepth*0.5
#pF.plotSnow(m, xptsG, yptsG, densityT, date_string=str(startDay-1), out=figpath+'/Snow/2layer/densityD'+driftP+extraStr+reanalysisP+varStr+'_sy'+str(year1)+'d'+str(startDay)+outStr+'T0', units_lab=r'kg/m3', minval=180, maxval=360, base_mask=0, norm=0, cmap_1=cm.viridis)
# Loop over days
for x in range(numDays-1):
day = x+startDay
if (day>=numDaysYear1):
# If day goes beyond the number of days in initial year, jump to the next year
day=day-numDaysYear1
yearCurrent=year2
print ('Day of year:', day)
print ('Date:', dates[x])
#-------- Load daily data
iceConcDayG, precipDayG, driftGdayG, windDayG, tempDayG =loadData(yearCurrent, day, precipVar, windVar, concVar, driftVar, dxStr, extraStr)
#-------- Apply CloudSat scaling if used
if scaleCS:
currentMonth = doyToMonth(day, yearCurrent) # get current month
scalingFactor = monthlyScalingFactors.loc[currentMonth,:,:] # get scaling factor for current month
# apply scaling to current day's precipitation
precipDayG = applyScaling(precipDayG, scalingFactor,scaling_type='mul').values
#-------- Calculate snow budgets
calcBudget(xptsG, yptsG, snowDepths, iceConcDayG, precipDayG, driftGdayG, windDayG, tempDayG,
density, precipDays, iceConcDays, windDays, tempDays, snowAcc, snowOcean, snowAdv,
snowDiv, snowLead, snowAtm, snowWindPackLoss, snowWindPackGain, snowWindPack, region_maskG, dx, x, day,
densityType=densityTypeT, dynamicsInc=dynamicsInc, leadlossInc=leadlossInc, windpackInc=windpackInc, atmlossInc=atmlossInc)
if (plotdaily==1):
cF.plot_gridded_cartopy(lonG, latG, snowDepths[x+1, 0]+snowDepths[x+1, 1], proj=ccrs.NorthPolarStereo(central_longitude=-45), date_string='', out=figpath+'daily_snow_depths/snowTot_'+saveStrNoDate+str(x), units_lab='m', varStr='Snow depth', minval=0., maxval=0.6)
#------ Load last data
iceConcDayG, precipDayG, _, windDayG, tempDayG =loadData(yearCurrent, day+1, precipVar, windVar, concVar, driftVar, dxStr, extraStr)
precipDays[x+1]=precipDayG
iceConcDays[x+1]=iceConcDayG
windDays[x+1]=windDayG
tempDays[x+1]=tempDayG
if (saveData==1):
# Output snow budget terms to netcdf datafiles
cF.OutputSnowModelRaw(savePath, saveStr, snowDepths, density, precipDays, iceConcDays, windDays, snowAcc, snowOcean, snowAdv, snowDiv, snowLead, snowAtm, snowWindPack)
cF.OutputSnowModelFinal(savePath, 'NESOSIMv11_'+dateOut, lonG, latG, xptsG, yptsG, snowDepths[:, 0]+snowDepths[:, 1], (snowDepths[:, 0]+snowDepths[:, 1])/iceConcDays, density, iceConcDays, precipDays, windDays, tempDays, dates)
if (plotBudgets==1):
# Plot final snow budget terms
cF.plot_budgets_cartopy(lonG, latG, precipDayG, windDayG, snowDepths[x+1], snowOcean[x+1], snowAcc[x+1], snowDiv[x+1], \
snowAdv[x+1], snowLead[x+1], snowAtm[x+1], snowWindPack[x+1], snowWindPackLoss[x+1], snowWindPackGain[x+1], density[x+1], dates[-1], figpath, totalOutStr=saveStr)
