def read_define_bounds_netcdf(file, band_conversion, extent):
"""
Using the NetCDF file:
Read with ncdf4 to extract information about the data extent
Regrid to rectangular projection using remap
ATTENTION: If the files are from the Operational Mode (starting December
2017), remap.py should be altered! (line 15)
"""
print('Reading NetCDF file ' + file)
nc = Dataset(file)
# Visualization extent for Full Disk
# geo_extent = nc.variables['geospatial_lat_lon_extent']
# min_lon = float(geo_extent.geospatial_westbound_longitude)
# max_lon = float(geo_extent.geospatial_eastbound_longitude)
# min_lat = float(geo_extent.geospatial_southbound_latitude)
# max_lat = float(geo_extent.geospatial_northbound_latitude)
# extent = [min_lon, min_lat, max_lon, max_lat]
resolution = 2
# Image extent required for the reprojection
H = nc.variables['goes_imager_projection'].perspective_point_height
x1 = nc.variables['x_image_bounds'][0] * H # x1 = -5434894.885056
x2 = nc.variables['x_image_bounds'][1] * H # x2 = 5434894.885056
y1 = nc.variables['y_image_bounds'][1] * H # y1 = -5434894.885056
y2 = nc.variables['y_image_bounds'][0] * H # y2 = 5434894.885056
grid = remap(file, extent, resolution, x1, y1, x2, y2)
data = grid.ReadAsArray() + band_conversion
return data, extent
def read_and_save_image(file_name):
# Parameters
extent = [-90.0, -70.0, -20, 10.0]
resolution = 2.0
# Call the reprojection funcion
grid = remap(file_name, extent, resolution, 'HDF5')
data = grid.ReadAsArray()
min_data = np.min(data)
max_data = np.max(data)
# Plot the Data
bmap = Basemap(llcrnrlon=extent[0],
llcrnrlat=extent[1],
urcrnrlon=extent[2],
urcrnrlat=extent[3],
epsg=4326)
bmap.imshow(data,
origin='upper',
cmap='gist_ncar',
vmin=min_data,
vmax=max_data)
# Save png
DPI = 300
png_file_name = file_name[:-2] + 'png'
plt.savefig(png_file_name, dpi=DPI, bbox_inches='tight', pad_inches=0)
return png_file_name
def create_tiff(filename, output, bbox):
# config
GDAL_DATA_TYPE = gdal.GDT_Float64
GEOTIFF_DRIVER_NAME = 'GTiff'
SPATIAL_REFERENCE_SYSTEM_WKID = 4326
BBOX = bbox
FILE = filename
data = remap(FILE, BBOX, 2, 'NETCDF').ReadAsArray()
xres = abs(BBOX[0] - BBOX[2]) / data.shape[1]
yres = abs(BBOX[1] - BBOX[3]) / data.shape[0]
geotransform = (BBOX[0], xres, 0, BBOX[3], 0, -yres)
# create the 3-band raster file
dst_ds = gdal.GetDriverByName(GEOTIFF_DRIVER_NAME)\
.Create(output, data.shape[1], data.shape[0], 1, GDAL_DATA_TYPE)
dst_ds.SetGeoTransform(geotransform) # specify coords
srs = osr.SpatialReference() # establish encoding
srs.ImportFromEPSG(SPATIAL_REFERENCE_SYSTEM_WKID)
dst_ds.SetProjection(srs.ExportToWkt()) # export coords to file
dst_ds.GetRasterBand(1).WriteArray(data) # write a-band to the raster
dst_ds.FlushCache() # write to disk
dst_ds = None
print(output, 'created')
def test_remap():
image = cv2.imread(sys.argv[1])
cv2.imshow('source',image)
source_image = torch.Tensor(image).transpose(0,2).transpose(1,2)
intrinsics_s = torch.Tensor([[718.8560,0,607.1928],[0,718.8960,185.2152],[0,0,1]])
intrinsics_t = torch.Tensor([[800,0,320],[0,800,240],[0,0,1]])
target_image_shape = (640,480)
target_image =remap(source_image.unsqueeze(0),intrinsics_s.unsqueeze(0),intrinsics_t.unsqueeze(0),target_image_shape)
target_image_show = target_image[0].squeeze().transpose(0,2).transpose(0,1)
cv2.imshow('target',np.array(target_image_show)/255)
cv2.waitKey()
def compute_angles(arm, target_frame):
from remap import remap
import math
# compute the inverse Kinematics and store the angles for each joint
#
angles = arm.inverse_kinematics(target_frame)
# convert the angle from radian to degrees
#
for (i, _) in enumerate(angles):
angles[i] = math.degrees(angles[i])
# remap the coordinates for each servo using Brandon's lOgIc
# remap each angle using Brandon's LoGic
# print each angle in degrees and position for debugging
#
for i in range(4, -1, -1):
angles[i] = remap(angles=angles, id=i)
print("angle({}) = {} deg, {} pos".format(i, angles[i], int(angles[i]/0.24)))
return angles
dst_dir = './'
print('Build IC file from the following file:')
print(file)
print(' ')
src_grd = pycnal_toolbox.CGrid_GLORYS.get_nc_CGrid_GLORYS(
'/archive/u1/uaf/kate/GLORYS/GL2V1_mesh_mask_new.nc',
name='GLORYS',
area='npolar',
ystart=690)
dst_grd = pycnal.grid.get_ROMS_grid('ARCTIC2')
# remap
zeta = remap(file, 'sossheig', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'iicethic', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'ileadfra', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'iicetemp', src_grd, dst_grd, dst_dir=dst_dir)
dst_grd = pycnal.grid.get_ROMS_grid('ARCTIC2', zeta=zeta)
remap(file, 'votemper', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'vosaline', src_grd, dst_grd, dst_dir=dst_dir)
#remap_uv(file, src_grd, dst_grd, dst_dir=dst_dir)
# merge file
ic_file = dst_dir + file.rsplit('/')[-1][:-4] + '_ic_' + dst_grd.name + '.nc'
out_file = dst_dir + file.rsplit(
'/')[-1][:-4] + '_sossheig_ic_' + dst_grd.name + '.nc'
command = ('ncks', '-a', '-O', out_file, ic_file)
subprocess.call(command)
from remap import remap
from remap_uv import remap_uv
file = '/archive/u1/uaf/kate/SODA/1985/SODA_2.1.6_19841230-19850104.cdf'
dst_dir='./'
print 'Build IC file from the following file:'
print file
print ' '
src_grd = pyroms_toolbox.BGrid_SODA.get_nc_BGrid_SODA('/archive/u1/uaf/kate/SODA/SODA_grid.cdf', name='SODA_2.1.6_ARC', area='global')
dst_grd = pyroms.grid.get_ROMS_grid('ARCTIC')
# remap
zeta = remap(file, 'ssh', src_grd, dst_grd, dst_dir=dst_dir)
dst_grd = pyroms.grid.get_ROMS_grid('ARCTIC', zeta=zeta)
remap(file, 'temp', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'salt', src_grd, dst_grd, dst_dir=dst_dir)
remap_uv(file, src_grd, dst_grd, dst_dir=dst_dir)
# merge file
ic_file = dst_dir + file.rsplit('/')[-1][:-4] + '_ic_' + dst_grd.name + '.nc'
out_file = dst_dir + file.rsplit('/')[-1][:-4] + '_ssh_ic_' + dst_grd.name + '.nc'
command = ('ncks', '-a', '-O', out_file, ic_file)
#subprocess.check_call(command)
subprocess.call(command)
os.remove(out_file)
out_file = dst_dir + file.rsplit('/')[-1][:-4] + '_temp_ic_' + dst_grd.name + '.nc'
command = ('ncks', '-a', '-A', out_file, ic_file)
num_maps = 1
map_method = 'bilinear'
pyroms.remapping.compute_remap_weights(grid1_file, grid2_file, \
interp_file1, interp_file2, map1_name, \
map2_name, num_maps, map_method)
# load the grid
src_grd = get_nc_Grid_HYCOM(src_grd_file)
dst_grd = pyroms.grid.get_ROMS_grid(grid_name)
# Triggering of variables remapping
zero=731;l_time=732 #2006-01-01 - 731; 2009-01-01 - 1827
zeta = remap(zero,l_time, file, 'ssh', src_grd, dst_grd, grid_name)
remap(zero,l_time, file, 'votemper', src_grd, dst_grd, grid_name)
remap(zero,l_time, file, 'vosaline', src_grd, dst_grd, grid_name)
remap_uv(zero,l_time, file, src_grd, dst_grd, grid_name)
# merge file
ic_file = '/home/eivanov/coawst_data_prrocessing/Temporal/Input_files_to_ROMS/ChildRiver_Initial.nc'
try:
os.remove(ic_file)
except:
pass
out_file = 'ssh%s.nc' %(grid_name)
command = ('ncks', '-a', '-A', out_file, ic_file)
subprocess.check_call(command)
def plot(sistemas_convectivos_nc, glm_nc, local, extent, resolution=2):
if (local == 'Brasil'):
degrees = 10
label_fontsize = 50
elif (local == 'Sudeste'):
degrees = 5
label_fontsize = 8
else:
degrees = 2
resolution = 0.8
label_fontsize = 8
g16glm = Dataset(glm_nc, 'r')
nc = Dataset(sistemas_convectivos_nc)
# Get the latitude and longitude image bounds
geo_extent = nc.variables['geospatial_lat_lon_extent']
min_lon = float(geo_extent.geospatial_westbound_longitude)
max_lon = float(geo_extent.geospatial_eastbound_longitude)
min_lat = float(geo_extent.geospatial_southbound_latitude)
max_lat = float(geo_extent.geospatial_northbound_latitude)
# Choose the visualization extent (min lon, min lat, max lon, max lat)
#extent = [-45.0, -24.5, -39.0, -20.7]
# Choose the image resolution (the higher the number the faster the processing is)
#resolution = 0.8 #2
# Calculate the image extent required for the reprojection
H = nc.variables['goes_imager_projection'].perspective_point_height
x1 = nc.variables['x_image_bounds'][0] * H
x2 = nc.variables['x_image_bounds'][1] * H
y1 = nc.variables['y_image_bounds'][1] * H
y2 = nc.variables['y_image_bounds'][0] * H
# Call the reprojection funcion
grid = remap(sistemas_convectivos_nc, extent, resolution, x1, y1, x2, y2)
tipos = ["todos", "event", "group", "flash"]
for formato in range(4):
glm_variables = [False, False, False, False]
glm_variables[formato] = True
#print(glm_variables)
# Read the data returned by the function ==============================================================
# If it is an IR channel subtract 273.15 to convert to ° Celsius
data = grid.ReadAsArray() - 273.15
# Make pixels outside the footprint invisible
data[data <= -180] = np.nan
# Define the size of the saved picture =================================================================
DPI = 150
fig = plt.figure(figsize=(data.shape[1] / float(DPI),
data.shape[0] / float(DPI)),
frameon=False,
dpi=DPI)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
ax = plt.axis('off')
# Plot the Data =======================================================================================
# Create the basemap reference for the Rectangular Projection
bmap = Basemap(llcrnrlon=extent[0],
llcrnrlat=extent[1],
urcrnrlon=extent[2],
urcrnrlat=extent[3],
epsg=4326)
# Draw the countries and Brazilian states shapefiles
bmap.readshapefile('/home/cendas/GOES16_WS_Rodrigo/GLM/src/BRA_adm1',
'BRA_adm1',
linewidth=0.50,
color='#000000')
if (local == 'Brasil'):
bmap.readshapefile(
'/home/cendas/GOES16_WS_Rodrigo/GLM/src/ne_10m_coastline',
'ne_10m_coastline',
linewidth=0.50,
color='#000000')
#ne_10m_coastline
# Draw parallels and meridians
if (not (local == 'Brasil')):
bmap.drawparallels(np.arange(-90.0, 90.0, degrees),
linewidth=0.3,
dashes=[4, 4],
color='white',
labels=[True, False, False, True],
fmt='%g',
labelstyle="+/-",
size=8)
bmap.drawmeridians(np.arange(0.0, 360.0, degrees),
linewidth=0.3,
dashes=[4, 4],
color='white',
labels=[True, False, False, True],
fmt='%g',
labelstyle="+/-",
size=8)
else:
bmap.drawparallels(np.arange(-90.0, 90.0, degrees),
linewidth=0.3,
dashes=[4, 4],
color='white',
labels=[True, False, False, True],
fmt='%g',
labelstyle="+/-",
size=30)
bmap.drawmeridians(np.arange(0.0, 360.0, degrees),
linewidth=0.3,
dashes=[4, 4],
color='white',
labels=[True, False, False, True],
fmt='%g',
labelstyle="+/-",
size=30)
#Split the dataset with temperatures above and below -20°C
temp = -80
tempAbove = masked_array(data, data < temp)
tempBelow = masked_array(data, data >= temp)
# Converts a CPT file to be used in Python
cptSquareRoot = loadCPT(
'/home/cendas/GOES16_WS_Rodrigo/GLM/src/Square Root Visible Enhancement.cpt'
)
# Makes a linear interpolation
cpt_convert_SquareRoot = LinearSegmentedColormap('cpt', cptSquareRoot)
# Plot the GOES-16 channel with the converted CPT colors (you may alter the min and max to match your preference)
plot_SquareRoot = bmap.imshow(tempAbove,
origin='upper',
cmap=cpt_convert_SquareRoot,
vmin=-80,
vmax=100)
# ===================== LEGENDA ==========================
# Get the unit based on the channel. If channels 1 trough 6 is Albedo. If channels 7 to 16 is BT.
Unit = "Cloud Top Temperature [°C]"
# Choose a title for the plot
Title = " Geostationary Lightning Mapper (GLM) - GOES Satellite"
Latitude = "Latitude"
Longitude = "Longitude"
# Add a black rectangle in the bottom to insert the image description
lon_difference = (extent[2] - extent[0]) # Max Lon - Min Lon
# Add the image description inside the black rectangle
lat_difference = (extent[3] - extent[1]) # Max lat - Min lat
if (not (local == 'Brasil')):
#Labels and its positions #
plt.text(extent[0] + lon_difference * 0.5,
extent[3] + lat_difference * 0.035,
Title,
horizontalalignment='center',
color='black',
size=9)
plt.text(extent[0] + lon_difference * 0.5,
extent[3] + lat_difference * 0.065,
" ",
horizontalalignment='center',
color='black',
size=10)
plt.text(extent[0] + lon_difference * 0.5,
extent[1] - lat_difference * 0.11,
Longitude,
horizontalalignment='center',
color='black',
size=10)
plt.text(extent[0] + lon_difference * 0.5,
extent[1] - lat_difference * 0.15,
" ",
horizontalalignment='center',
color='black',
size=18)
plt.text(extent[0] - lon_difference * 0.15,
extent[1] + lat_difference * 0.5,
Latitude,
verticalalignment='center',
rotation="vertical",
color='black',
size=10)
else:
plt.text(extent[0] + lon_difference * 0.5,
extent[3] + lat_difference * 0.035,
Title,
horizontalalignment='center',
color='black',
size=40)
plt.text(extent[0] + lon_difference * 0.5,
extent[3] + lat_difference * 0.065,
" ",
horizontalalignment='center',
color='black',
size=10)
plt.text(extent[0] + lon_difference * 0.5,
extent[1] - lat_difference * 0.11,
Longitude,
horizontalalignment='center',
color='black',
size=40)
plt.text(extent[0] + lon_difference * 0.5,
extent[1] - lat_difference * 0.15,
" ",
horizontalalignment='center',
color='black',
size=18)
plt.text(extent[0] - lon_difference * 0.15,
extent[1] + lat_difference * 0.5,
Latitude,
verticalalignment='center',
rotation="vertical",
color='black',
size=40)
# ========================================
if (glm_variables[0]): #Todos
# Get Events, Group and flash from Glm file
event_lat = g16glm.variables['event_lat'][:]
event_lon = g16glm.variables['event_lon'][:]
group_lat = g16glm.variables['group_lat'][:]
group_lon = g16glm.variables['group_lon'][:]
flash_lat = g16glm.variables['flash_lat'][:]
flash_lon = g16glm.variables['flash_lon'][:]
# Plot events as large blue dots
event_x, event_y = bmap(event_lon, event_lat)
bmap.plot(event_x, event_y, 'bo', markersize=7, label='Events')
# Plot groups as medium green dots
group_x, group_y = bmap(group_lon, group_lat)
bmap.plot(group_x, group_y, 'go', markersize=3, label='Group')
# Plot flashes as small red dots
flash_x, flash_y = bmap(flash_lon, flash_lat)
bmap.plot(flash_x, flash_y, 'ro', markersize=1, label='Flash')
plt.legend(fontsize=label_fontsize, loc=4)
else:
if (glm_variables[1]):
# Get Events from Glm file
event_lat = g16glm.variables['event_lat'][:]
event_lon = g16glm.variables['event_lon'][:]
# Plot events as large blue dots
event_x, event_y = bmap(event_lon, event_lat)
bmap.plot(event_x, event_y, 'bo', markersize=7, label='Events')
if (glm_variables[2]):
# Get Group from Glm file
group_lat = g16glm.variables['group_lat'][:]
group_lon = g16glm.variables['group_lon'][:]
# Plot groups as medium green dots
group_x, group_y = bmap(group_lon, group_lat)
bmap.plot(group_x, group_y, 'go', markersize=3, label='Group')
if (glm_variables[3]):
# Get Flash from Glm file
flash_lat = g16glm.variables['flash_lat'][:]
flash_lon = g16glm.variables['flash_lon'][:]
# Plot flashes as small red dots
flash_x, flash_y = bmap(flash_lon, flash_lat)
bmap.plot(flash_x, flash_y, 'ro', markersize=1, label='Flash')
plt.legend(fontsize=label_fontsize, loc=4)
#plt.show()
Start = glm_nc[glm_nc.find('_s') + 1:glm_nc.find("_e") - 3]
year = int(Start[1:5])
dayjulian = int(
Start[5:8]) - 1 # Subtract 1 because the year starts at "0"
dayconventional = datetime.datetime(year, 1, 1) + datetime.timedelta(
dayjulian) # Convert from julian to conventional
month = dayconventional.strftime('%m')
day = dayconventional.strftime('%d')
hour = int(Start[8:12])
plt.savefig('/home/cendas/GOES16_WS_Rodrigo/GLM/Output/' + local +
'/glm_' + str(year) + str(month) + str(day) + '_' +
str(hour) + '_' + tipos[formato] + '.png',
dpi=DPI,
pad_inches=0,
transparent=True,
bbox_inches='tight')
plt.close()
file = '/nfs/P1/Data/SODA/SODA_2.1.6/SODA_2.1.6_20031231-20040105.cdf'
dst_dir = './'
print('Build IC file from the following file:')
print(file)
print(' ')
src_grd = pycnal_toolbox.BGrid_SODA.get_nc_BGrid_SODA(
'/nfs/P1/Data/SODA/SODA_2.1.6/SODA_grid.cdf',
name='SODA_2.1.6',
area='npolar',
ystart=240)
dst_grd = pycnal.grid.get_ROMS_grid('ARCTIC2')
# remap
zeta = remap(file, 'ssh', src_grd, dst_grd, dst_dir=dst_dir)
dst_grd = pycnal.grid.get_ROMS_grid('ARCTIC2', zeta=zeta)
remap(file, 'temp', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'salt', src_grd, dst_grd, dst_dir=dst_dir)
remap_uv(file, src_grd, dst_grd, dst_dir=dst_dir)
# merge file
ic_file = dst_dir + file.rsplit('/')[-1][:-4] + '_ic_' + dst_grd.name + '.nc'
out_file = dst_dir + file.rsplit(
'/')[-1][:-4] + '_ssh_ic_' + dst_grd.name + '.nc'
command = ('ncks', '-a', '-O', out_file, ic_file)
#subprocess.check_call(command)
subprocess.call(command)
os.remove(out_file)
out_file = dst_dir + file.rsplit(
from remap import remap
from remap_uv import remap_uv
file = '/archive/u1/uaf/kate/GLORYS/GLORYS2V3_1dAV_19980101_19980102.nc'
dst_dir='./'
print 'Build IC file from the following file:'
print file
print ' '
src_grd = pyroms_toolbox.CGrid_GLORYS.get_nc_CGrid_GLORYS('/archive/u1/uaf/kate/GLORYS/GL2V1_mesh_mask_new.nc', name='GLORYS', area='npolar', ystart=690)
dst_grd = pyroms.grid.get_ROMS_grid('ARCTIC2')
# remap
zeta = remap(file, 'sossheig', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'iicethic', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'ileadfra', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'iicetemp', src_grd, dst_grd, dst_dir=dst_dir)
dst_grd = pyroms.grid.get_ROMS_grid('ARCTIC2', zeta=zeta)
remap(file, 'votemper', src_grd, dst_grd, dst_dir=dst_dir)
remap(file, 'vosaline', src_grd, dst_grd, dst_dir=dst_dir)
#remap_uv(file, src_grd, dst_grd, dst_dir=dst_dir)
# merge file
ic_file = dst_dir + file.rsplit('/')[-1][:-4] + '_ic_' + dst_grd.name + '.nc'
out_file = dst_dir + file.rsplit('/')[-1][:-4] + '_sossheig_ic_' + dst_grd.name + '.nc'
command = ('ncks', '-a', '-O', out_file, ic_file)
subprocess.call(command)
os.remove(out_file)
import pyroms; from Get_grid import Get_grid_Mercator_Nest; from Get_grid import Get_grid_Mercator_Parent; from make_remap_grid_file import make_remap_grid_file; from remap import remap
src_grd_file = '/media/sf_Swap-between-windows-linux/New_Grid/TEMP_SALT_CURR_2004_2014.nc'
dst_grd_file = 'Temp_Bot_One_Parent.nc'
srcgrd = Get_grid_Mercator_Parent(src_grd_file)
dstgrd = pyroms.grid.get_ROMS_grid('COARSEST')
make_remap_grid_file(srcgrd)
pyroms.remapping.make_remap_grid_file(dstgrd, Cpos='rho'); pyroms.remapping.make_remap_grid_file(dstgrd, Cpos='u'); pyroms.remapping.make_remap_grid_file(dstgrd, Cpos='v')
# compute remap weights input namelist variables for bilinear remapping at rho points
grid1_file = 'remap_grid_MERCATOR_PARENT_t.nc'
grid2_file = 'remap_grid_COARSEST_rho.nc'
interp_file1 = 'remap_weights_MERCATOR_PARENT_to_COARSEST_bilinear_t_to_rho.nc'
interp_file2 = 'remap_weights_COARSEST_to_MERCATOR_PARENT_bilinear_rho_to_t.nc'
map1_name = 'MERCATOR_PARENT to COARSEST Bilinear Mapping'
map2_name = 'COARSEST to MERCATOR_PARENT Bilinear Mapping'
num_maps = 1
map_method = 'bilinear'
pyroms.remapping.compute_remap_weights(grid2_file, grid1_file, interp_file2, interp_file1, map2_name, map1_name, num_maps, map_method)
tart=0; tend=1095*24
wts='remap_weights_COARSEST_to_MERCATOR_PARENT_bilinear_rho_to_t.nc'
remap(tart, tend, dst_grd_file, 'temp', wts, dstgrd, srcgrd, src_grd_file, dst_dir='./')
def create_dataframe_abi2(path_abi, time):
files = [os.path.join(path_abi, file) for file in os.listdir(path_abi)]
df = gpd.read_file(os.path.join(HERE, 'shapefiles/estadosl_2007.shp'))
print(files[0])
data = dict()
data['start_scan'] = []
data['end_scan'] = []
data['year'] = []
data['julian_day'] = []
data['hour'] = []
data['minute'] = []
data['state'] = []
data['min'] = []
data['mean'] = []
data['std'] = []
data['max'] = []
data['timestamp'] = []
timestamp = time['t']
for file in files:
if not file.endswith('.nc'):
continue
filename = file.split('/')[-1]
start_scan = filename[27:41]
end_scan = filename[43:57]
y = start_scan[:4]
jd = start_scan[4:7]
h = start_scan[7:9]
m = start_scan[9:11]
for _, state in df.iterrows():
print('Geting values from %s...' % state['NOMEUF2'])
bbox = state['geometry'].bounds
grid = remap(file, bbox, 1, 'NETCDF').ReadAsArray()
data['min'].append(np.min(grid))
data['mean'].append(np.mean(grid))
data['max'].append(np.max(grid))
data['std'].append(np.std(grid))
data['state'].append(state['NOMEUF2'])
data['start_scan'].append(start_scan)
data['end_scan'].append(end_scan)
data['year'].append(y)
data['julian_day'].append(jd)
data['hour'].append(h)
data['minute'].append(m)
data['timestamp'].append(timestamp)
output_csv = '/home/adriano/earth-observation/.data/csv/abi' #path_abi.replace('nc', 'csv')
csv_filename = '%s__' % time['t']
csv_filename += '%s_s%s%s%s%s' % ('ABI-L2-CMIPF', time['y'], time['d'],
time['h'], time['m'][0])
csv_filename += '_e%s%s%s%s.csv' % (time['y'], time['d'], time['h'],
time['m'][-1])
if not os.path.exists(output_csv):
os.makedirs(output_csv)
df = pd.DataFrame(data)
filename = os.path.join(output_csv, csv_filename)
df.to_csv(filename, index=False)
print('CSV ABI file saved in ', filename)
def create_dataframe_abi(path_abi, time):
print('Creating ABI dataframe...')
files = [os.path.join(path_abi, file) for file in os.listdir(path_abi)]
df = load_states()
print(files[0])
data = dict()
data['start_scan'] = []
data['end_scan'] = []
data['year'] = []
data['julian_day'] = []
data['hour'] = []
data['minute'] = []
data['lon'] = []
data['lat'] = []
data['state'] = []
data['value'] = []
data['ind_x'] = []
data['ind_y'] = []
data['rows'] = []
data['cols'] = []
data['timestamp'] = []
timestamp = time['t']
for file in files:
filename = file.split('/')[-1]
start_scan = filename[27:41]
end_scan = filename[43:57]
y = start_scan[:4]
jd = start_scan[4:7]
h = start_scan[7:9]
m = start_scan[9:11]
for _, state in df.iterrows():
print('Geting values from %s...' % state['NOMEUF2'])
bbox = state['geometry'].bounds
grid = remap(file, bbox, 1, 'NETCDF').ReadAsArray()
lons = np.linspace(bbox[0], bbox[2], grid.shape[1])
lats = np.linspace(bbox[1], bbox[3], grid.shape[0])
rows = grid.shape[0]
cols = grid.shape[1]
for x in range(rows):
for y in range(cols):
data['rows'].append(rows)
data['cols'].append(cols)
data['ind_x'].append(x)
data['ind_y'].append(y)
data['lon'].append(lons[y])
data['lat'].append(lats[x])
data['value'].append(grid[x][y])
data['state'].append(state['NOMEUF2'])
data['start_scan'].append(start_scan)
data['end_scan'].append(end_scan)
data['year'].append(y)
data['julian_day'].append(jd)
data['hour'].append(h)
data['minute'].append(m)
data['timestamp'].append(timestamp)
output_csv = path_abi.replace('nc', 'csv')
csv_filename = '%s__' % time['t']
csv_filename += '%s_s%s%s%s%s' % ('ABI-L2-CMIPF', time['y'], time['d'],
time['h'], time['m'][0])
csv_filename += '_e%s%s%s%s.csv' % (time['y'], time['d'], time['h'],
time['m'][-1])
if not os.path.exists(output_csv):
os.makedirs(output_csv)
df = pd.DataFrame(data)
filename = os.path.join(output_csv, csv_filename)
df.to_csv(filename, index=False)
def func(dst_grd_file,order):
if order=='PARENT':
srcgrd1 = Get_grid_ODYSSEA_Parent(src_grd_file1)
srcgrd2 = Get_grid_Mercator_Parent(src_grd_file2)
grid='COARSEST'
dstgrd = pyroms.grid.get_ROMS_grid(grid)
pyroms.remapping.make_remap_grid_file(dstgrd, Cpos='rho')
elif order=='NEST':
srcgrd1 = Get_grid_ODYSSEA_Nest(src_grd_file1)
srcgrd2 = Get_grid_Mercator_Nest(src_grd_file2)
grid='FINER'
dstgrd = pyroms.grid.get_ROMS_grid(grid)
pyroms.remapping.make_remap_grid_file_2(dstgrd, Cpos='rho')
#### Uncomment if you want to compute weights between two grid ####
make_remap_grid_file(srcgrd1)
make_remap_grid_file(srcgrd2)
# compute remap weights input namelist variables for bilinear remapping at rho points
grid1_file = 'remap_grid_ODYSSEA_%s_t.nc' %(order)
grid2_file = 'remap_grid_%s_rho.nc' %(grid)
interp_file1 = 'remap_weights_ODYSSEA_%s_to_%s_bilinear_t_to_rho.nc' %(order,grid)
interp_file2 = 'remap_weights_%s_to_ODYSSEA_%s_bilinear_rho_to_t.nc' %(grid,order)
map1_name = 'ODYSSEA_%s to %s Bilinear Mapping' %(order,grid)
map2_name = '%s to ODYSSEA_%s Bilinear Mapping' %(grid,order)
num_maps = 1
map_method = 'bilinear'
pyroms.remapping.compute_remap_weights(grid2_file, grid1_file, interp_file2, interp_file1, map2_name, map1_name, num_maps, map_method)
grid1_file = 'remap_grid_MERCATOR_%s_t.nc' %(order)
grid2_file = 'remap_grid_%s_rho.nc' %(grid)
interp_file1 = 'remap_weights_MERCATOR_%s_to_%s_bilinear_t_to_rho.nc' %(order,grid)
interp_file2 = 'remap_weights_%s_to_MERCATOR_%s_bilinear_rho_to_t.nc' %(grid,order)
map1_name = 'MERCATOR_%s to %s Bilinear Mapping' %(order,grid)
map2_name = '%s to MERCATOR_%s Bilinear Mapping' %(grid,order)
num_maps = 1
map_method = 'bilinear'
pyroms.remapping.compute_remap_weights(grid2_file, grid1_file, interp_file2, interp_file1, map2_name, map1_name, num_maps, map_method)
tt=365*3
dst_grd_file = '/home/eivanov/coawst_data_prrocessing/Temporal/Former/' + dst_grd_file
wts = 'remap_weights_%s_to_ODYSSEA_%s_bilinear_rho_to_t.nc' %(grid,order)
remap(tt, dst_grd_file, 'sst', wts, dstgrd, srcgrd1, src_grd_file1,order,grid)
wts = 'remap_weights_%s_to_MERCATOR_%s_bilinear_rho_to_t.nc' %(grid,order)
remap(tt, dst_grd_file, 'sbt', wts, dstgrd, srcgrd2, src_grd_file2,order,grid)
remap(tt, dst_grd_file, 'sss', wts, dstgrd, srcgrd2, src_grd_file2,order,grid)
remap(tt, dst_grd_file, 'sbs', wts, dstgrd, srcgrd2, src_grd_file2,order,grid)
remap(tt, dst_grd_file, 'ssu', wts, dstgrd, srcgrd2, src_grd_file2,order,grid)
remap(tt, dst_grd_file, 'ssv', wts, dstgrd, srcgrd2, src_grd_file2,order,grid)
remap(tt, dst_grd_file, 'sbu', wts, dstgrd, srcgrd2, src_grd_file2,order,grid)
remap(tt, dst_grd_file, 'sbv', wts, dstgrd, srcgrd2, src_grd_file2,order,grid)
a='sstODYSSEA_%s.nc' %(order)
b='%s_sstODYSSEA_%s.nc' %(dst_grd_file[:-3],order)
os.rename(a,b)
ic_file = dst_grd_file[:-3]+'_replottedMERCATOR_%s.nc' %(order)
cycle=['sbt','sss','sbs','ssu','ssv','sbu','sbv']
for i in range(len(cycle)):
out_file = '%sMERCATOR_%s.nc' %(cycle[i],order)
command = ('ncks', '-a', '-A', out_file, ic_file)
subprocess.check_call(command)
os.remove(out_file)
random.seed(42)
# Load stock data
eod_data = load_stock_data()
# Loop through all stocks returned
for key, data in eod_data.items():
# Process data
processed_data = process_data(data)
X, Y = [], []
for idx in range(0, len(processed_data)-WINDOW-FORECAST, STEP):
# Get data from window
hl = remap(np.array(processed_data['H-L'][idx:idx+WINDOW]), -1, 1)
co = remap(np.array(processed_data['C-O'][idx:idx+WINDOW]), -1, 1)
sma_3 = remap(np.array(processed_data['3day SMA'][idx:idx+WINDOW]), -1, 1)
sma_10 = remap(np.array(processed_data['10day SMA'][idx:idx+WINDOW]), -1, 1)
sma_30 = remap(np.array(processed_data['30day SMA'][idx:idx+WINDOW]), -1, 1)
std_dev = remap(np.array(processed_data['Std_dev'][idx:idx+WINDOW]), -1, 1)
rsi = remap(np.array(processed_data['RSI'][idx:idx+WINDOW]), -1, 1)
# Stack in array
x_i = np.column_stack((hl, co, sma_3, sma_10, sma_30, std_dev, rsi))
# Get forecast value
last_close = processed_data['C'][idx + WINDOW]
next_close = processed_data['C'][idx + WINDOW + FORECAST]
if last_close < next_close:
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
from netCDF4 import Dataset
from remap import remap
import geopandas as gpd
file = "data/2017/10/2/OR_ABI-L2-CMIPF-M3C02_G16_s20172750930388_e20172750941155_c20172750941226.nc"
#bbox = [-53.08059692, -25.25489044, -44.20065689, -19.78015137]
shapefile = 'shapes/Brazilian_States_Shape/BRA_adm1.shp'
shp = gpd.read_file(shapefile)
state = shp[shp['ADM1'] == 'SAO PAULO']
bbox = state.bounds.values[0]
grid = remap(file, bbox, 2, 'NETCDF')
data = grid.ReadAsArray()
#DPI = 150
#ax = plt.figure(figsize=(2000/float(DPI), 2000/float(DPI)), frameon=True, dpi=DPI)
bmap = Basemap(projection='merc',
llcrnrlon=bbox[0],
llcrnrlat=bbox[1],
urcrnrlon=bbox[2],
urcrnrlat=bbox[3] + 0.5,
epsg=4326,
resolution='i')
bmap.bluemarble()
bmap.imshow(data, cmap='jet', alpha=.8)
# Bocaina as a Center of the projection degrees = 5 # Choose the visualization extent (min lon, min lat, max lon, max lat) extent = [-45 - degrees, -23.7 - degrees, -43.4 + degrees, -22.5 + degrees] # Choose the image resolution (the higher the number the faster the processing is) resolution = 2 # Calculate the image extent required for the reprojection H = nc.variables['goes_imager_projection'].perspective_point_height x1 = nc.variables['x_image_bounds'][0] * H x2 = nc.variables['x_image_bounds'][1] * H y1 = nc.variables['y_image_bounds'][1] * H y2 = nc.variables['y_image_bounds'][0] * H # Call the reprojection funcion gridCH1 = remap(pathCH1, extent, resolution, x1, y1, x2, y2) gridCH2 = remap(pathCH2, extent, resolution, x1, y1, x2, y2) gridCH3 = remap(pathCH3, extent, resolution, x1, y1, x2, y2) # Read the data returned by the function R = gridCH2.ReadAsArray() G = gridCH3.ReadAsArray() B = gridCH1.ReadAsArray() # Apply range limits for each channel. RGB values must be between 0 and 1 R = np.clip(R, 0, 1) G = np.clip(G, 0, 1) B = np.clip(B, 0, 1) # Apply the gamma correction gamma = 2.2
for filein in filelst:
tag = filein.replace(data_dir + '/' + my_year + '/',
'').replace('soda3.3.1_5dy_ocean_reg_',
'').replace('.nc', '')
print '\nBuild OBC file for time %s' % tag
zeta_dst_file = dst_dir + dst_grd.name + '_IC_zeta_' + tag + '_' + src_grd.name + '.nc'
temp_dst_file = dst_dir + dst_grd.name + '_IC_temp_' + tag + '_' + src_grd.name + '.nc'
salt_dst_file = dst_dir + dst_grd.name + '_IC_salt_' + tag + '_' + src_grd.name + '.nc'
u_dst_file = dst_dir + dst_grd.name + '_IC_u_' + tag + '_' + src_grd.name + '.nc'
v_dst_file = dst_dir + dst_grd.name + '_IC_v_' + tag + '_' + src_grd.name + '.nc'
# remap ssh
zeta = remap('ssh',
filein,
src_grd,
dst_grd,
zeta_dst_file,
dst_dir=dst_dir)
# reload grid with zeta (more accurate)
dst_grd = pyroms.grid.get_ROMS_grid('ARCTIC4', zeta=zeta)
# regrid temp, salt and velocities
remap('temp', filein, src_grd, dst_grd, temp_dst_file, dst_dir=dst_dir)
remap('salt', filein, src_grd, dst_grd, salt_dst_file, dst_dir=dst_dir)
remap_uv(filein, src_grd, dst_grd, u_dst_file, v_dst_file, dst_dir=dst_dir)
# merge file
IC_file = dst_dir + dst_grd.name + '_IC_' + tag + '_' + src_grd.name + '.nc'
command1 = 'mv ' + zeta_dst_file + ' ' + IC_file
filein=data_dir + 'soda3.3.1_5dy_ocean_reg_' + tag + '.nc'
# load grids
src_grd = pyroms_toolbox.BGrid_GFDL.get_nc_BGrid_GFDL(data_dir + 'grid/SODA3_0.5deg_grid.nc', name='SODA3.3.1', xrange=(400, 500), yrange=(180, 280))
dst_grd = pyroms.grid.get_ROMS_grid('CCS')
print '\nBuild IC file from %s' %filein
zeta_dst_file = dst_dir + dst_grd.name + '_ic_zeta_' + tag + '_' + src_grd.name + '.nc'
temp_dst_file = dst_dir + dst_grd.name + '_ic_temp_' + tag + '_' + src_grd.name + '.nc'
salt_dst_file = dst_dir + dst_grd.name + '_ic_salt_' + tag + '_' + src_grd.name + '.nc'
u_dst_file = dst_dir + dst_grd.name + '_ic_u_' + tag + '_' + src_grd.name + '.nc'
v_dst_file = dst_dir + dst_grd.name + '_ic_v_' + tag + '_' + src_grd.name + '.nc'
# remap ssh
zeta = remap('ssh', filein, src_grd, dst_grd, zeta_dst_file, dst_dir=dst_dir)
# reload grid with zeta (more accurate)
dst_grd = pyroms.grid.get_ROMS_grid('CCS', zeta=zeta)
# regrid temp, salt and velocities
remap('temp',filein, src_grd, dst_grd, temp_dst_file, dst_dir=dst_dir)
remap('salt',filein, src_grd, dst_grd, salt_dst_file, dst_dir=dst_dir)
remap_uv(filein, src_grd, dst_grd, u_dst_file, v_dst_file, dst_dir=dst_dir)
# merge file
ic_file = dst_dir + dst_grd.name + '_ic_' + tag + '_' + src_grd.name + '.nc'
command1 = 'mv ' + zeta_dst_file + ' ' + ic_file
command2 = 'ncks -A ' + temp_dst_file + ' -o ' + ic_file
command3 = 'ncks -A ' + salt_dst_file + ' -o ' + ic_file
degrees = 0
# Choose the visualization extent (min lon, min lat, max lon, max lat)
extent = [-46 - degrees, -26 - degrees, -40 + degrees,
-19 + degrees] #Regiao MetroSerra
# Choose the image resolution (the higher the number the faster the processing is)
resolution = 0.8 #Standard max resolution without lose information = 2
# Calculate the image extent required for the reprojection
H = nc.variables['goes_imager_projection'].perspective_point_height
x1 = nc.variables['x_image_bounds'][0] * H
x2 = nc.variables['x_image_bounds'][1] * H
y1 = nc.variables['y_image_bounds'][1] * H
y2 = nc.variables['y_image_bounds'][0] * H
# Call the reprojection funcion
grid = remap(path, extent, resolution, x1, y1, x2, y2)
# Search for the GOES-16 channel in the file name
bandSetted = False
bands = ['M6C', 'M3C']
bandLenghts = [
'01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12',
'13', '14', '15', '16'
]
mode = 0
while not bandSetted:
Band = (path[path.find(bands[mode]) + 3:path.find("_G16")])
#if (len(Band) != 2) and (Band not in bandBounds):
if (Band not in bandLenghts):
mode += 1
file = "/archive/u1/uaf/kate/HYCOM/SCS/data/HYCOM_GLBa0.08_2015_001.nc"
data_dir = "/archive/u1/uaf/kate/HYCOM/SCS/data/"
dst_dir = "./"
print "Build IC file from the following file:"
print file
print " "
src_grd_file = data_dir + "../HYCOM_GLBa0.08_PALAU_grid.nc"
src_grd = pyroms_toolbox.Grid_HYCOM.get_nc_Grid_HYCOM(src_grd_file)
dst_grd = pyroms.grid.get_ROMS_grid("PALAU1")
# remap
zeta = remap(file, "ssh", src_grd, dst_grd, dst_dir=dst_dir)
dst_grd = pyroms.grid.get_ROMS_grid("PALAU1", zeta=zeta)
remap(file, "temp", src_grd, dst_grd, dst_dir=dst_dir)
remap(file, "salt", src_grd, dst_grd, dst_dir=dst_dir)
remap_uv(file, src_grd, dst_grd, dst_dir=dst_dir)
# merge file
ic_file = dst_dir + file.rsplit("/")[-1][:-3] + "_ic_" + dst_grd.name + ".nc"
out_file = dst_dir + file.rsplit("/")[-1][:-3] + "_ssh_ic_" + dst_grd.name + ".nc"
command = ("ncks", "-a", "-O", out_file, ic_file)
print command
subprocess.check_call(command)
os.remove(out_file)
out_file = dst_dir + file.rsplit("/")[-1][:-3] + "_temp_ic_" + dst_grd.name + ".nc"
command = ("ncks", "-a", "-A", out_file, ic_file)