def extract_area_rf_series(nc_f, lat_min, lat_max, lon_min, lon_max):
if not os.path.exists(nc_f):
raise IOError('File %s not found' % nc_f)
nc_fid = Dataset(nc_f, 'r')
times_len, times = extract_time_data(nc_f)
lats = nc_fid.variables['XLAT'][0, :, 0]
lons = nc_fid.variables['XLONG'][0, 0, :]
lon_min_idx = np.argmax(lons >= lon_min) - 1
lat_min_idx = np.argmax(lats >= lat_min) - 1
lon_max_idx = np.argmax(lons >= lon_max)
lat_max_idx = np.argmax(lats >= lat_max)
prcp = nc_fid.variables['RAINC'][:, lat_min_idx:lat_max_idx, lon_min_idx:
lon_max_idx] + nc_fid.variables[
'RAINNC'][:, lat_min_idx:lat_max_idx,
lon_min_idx:lon_max_idx]
diff = get_two_element_average(prcp)
nc_fid.close()
return diff, lats[lat_min_idx:lat_max_idx], lons[
lon_min_idx:lon_max_idx], np.array(times[0:times_len - 1])
def extract_variables(nc_f,
var_list,
lat_min,
lat_max,
lon_min,
lon_max,
lat_var='XLAT',
lon_var='XLONG',
time_var='Times'):
"""
extract variables from a netcdf file
:param nc_f:
:param var_list: comma separated string for variables / list of strings
:param lat_min:
:param lat_max:
:param lon_min:
:param lon_max:
:param lat_var:
:param lon_var:
:param time_var:
:return:
variables dict {var_key --> var[time, lat, lon], xlat --> [lat], xlong --> [lon], times --> [time]}
"""
if not os.path.exists(nc_f):
raise IOError('File %s not found' % nc_f)
nc_fid = Dataset(nc_f, 'r')
times = np.array([
''.join([y.decode() for y in x]) for x in nc_fid.variables[time_var][:]
])
lats = nc_fid.variables[lat_var][0, :, 0]
lons = nc_fid.variables[lon_var][0, 0, :]
lat_inds = np.where((lats >= lat_min) & (lats <= lat_max))
lon_inds = np.where((lons >= lon_min) & (lons <= lon_max))
vars_dict = {}
if isinstance(var_list, str):
var_list = var_list.replace(',', ' ').split()
# var_list = var_list.replace(',', ' ').split() if isinstance(var_list, str) else var_list
for var in var_list:
vars_dict[var] = nc_fid.variables[var][:, lat_inds[0], lon_inds[0]]
nc_fid.close()
vars_dict[time_var] = times
vars_dict[lat_var] = lats[lat_inds[0]]
vars_dict[lon_var] = lons[lon_inds[0]]
# todo: implement this archiving procedure
# if output is not None:
# logging.info('%s will be archied to %s' % (nc_f, output))
# ncks_extract_variables(nc_f, var_str, output)
return vars_dict
def extract_time_data(nc_f):
nc_fid = Dataset(nc_f, 'r')
times_len = len(nc_fid.dimensions['Time'])
try:
times = [''.join(x) for x in nc_fid.variables['Times'][0:times_len]]
except TypeError:
times = np.array([
''.join([y.decode() for y in x])
for x in nc_fid.variables['Times'][:]
])
nc_fid.close()
return times_len, times
def _create_nc_file_latlon2d(self,
a,
fname,
varname,
varunits=None,
fill=None):
self.log.debug('a={0}'.format(a))
self.log.debug('shape a = {0}'.format(a.shape))
assert len(a.shape) == 2
lat_dim, lon_dim = a.shape
rootgrp = Dataset(fname, "w", format="NETCDF4")
rootgrp.createDimension("lat", lat_dim)
rootgrp.createDimension("lon", lon_dim)
vals = rootgrp.createVariable(varname,
"f4",
dimensions=(
"lat",
"lon",
),
fill_value=fill)
lats = rootgrp.createVariable("lat", "f4", dimensions=("lat", ))
lons = rootgrp.createVariable("lon", "f4", dimensions=("lon", ))
vals[:, :] = a
lats[:] = np.linspace(self._minLatCent, self._maxLatCent, lat_dim)
lons[:] = np.linspace(self._minLonCent, self._maxLonCent, lon_dim)
if varunits is not None:
vals.units = varunits
lats.units = "degrees north"
lons.units = "degrees east"
rootgrp.close()
def check_product_can_be_opened(self):
"""
:rtype : Dataset
"""
try:
dataset = Dataset(self.source_pathname)
dataset.set_auto_maskandscale(False)
self.report['product_can_be_opened_check'] = float(0)
return dataset
except:
self.report['product_can_be_opened_check'] = float(1)
filename = os.path.basename(self.source_pathname)
self.report['product_can_be_opened_check_failed_for'] = filename
raise VerificationError
def check_product_can_be_opened(self):
"""
:rtype : Dataset
"""
try:
dataset = Dataset(self.source_pathname)
dataset.set_auto_maskandscale(False)
self.report['product_can_be_opened_check'] = float(0)
return dataset
except:
self.report['product_can_be_opened_check'] = float(1)
filename = os.path.basename(self.source_pathname)
self.report['product_can_be_opened_check_failed_for'] = filename
raise VerificationError
def transfer_netCDF_file(cfa, part_n):
"""Get a netCDF file from the partition info"""
part = cfa["Partitions"][part_n]
suba = part["subarray"]
file_name = os.path.join(cfa["base"], suba["file"])
file_details = get_netCDF_file_details(file_name, 'r')
# open the netCDF file
if file_details.memory == "":
fh = Dataset(file_details.filename,
mode=file_details.filemode,
diskless=False)
else:
fh = Dataset(file_details.filename,
mode=file_details.filemode,
diskless=True,
memory=file_details.memory)
def __init__(self, metadataFile, csvFile, ncOutput):
Log().set_log_info("[Begin] conversion to NetCDF of: " + metadataFile +
" " + csvFile + " " + ncOutput)
self.init_elements(metadataFile, csvFile, ncOutput)
if not os.path.exists(self.ncOutput):
try:
self.ncFile = Dataset(self.ncOutput,
'w',
format='NETCDF' + self.version)
except:
Log().set_log_error(
"The netCDF_version is wrong. Assigning the default value(netCDF4_CLASSIC) to "
"netCDF_version")
self.version = '4_CLASSIC'
self.ncFile = Dataset(self.ncOutput,
'w',
format='NETCDF' + self.version)
self.create_netcdf()
else: # Append start
self.ncFile = Dataset(self.ncOutput, 'r+')
self.append_netcdf()
self.metadata.globalAttributes.max_min_attribute(self.ncFile)
self.ncFile.close()
Log().set_log_info("[Finished] conversion to NetCDF of : " +
metadataFile + " " + csvFile + " " + ncOutput)
def basin_avg_netcdf(netcdf_file, shp_file, mask_file):
data_netcdf = Dataset(netcdf_file, 'r') # reads the netCDF file
print(data_netcdf)
# get all variable names
print(data_netcdf.variables.keys())
temp_lat = data_netcdf.variables['lat'] # temperature variable
temp_lon = data_netcdf.variables['lon'] # temperature variable
for d in data_netcdf.dimensions.items():
print(d)
x, y = data_netcdf.variables['x'], data_netcdf.variables['y']
x = data_netcdf.variables['x'][:]
y = data_netcdf.variables['y'][:]
lx = list(x)
ly = list(y)
print(all(ix < jx for ix, jx in zip(lx, lx[1:])))
print(all(iy > jy for iy, jy in zip(ly, ly[1:])))
lons = data_netcdf.variables['lon'][:]
lats = data_netcdf.variables['lat'][:]
crs_pro_str = '+proj=lcc +lat_1=25 +lat_2=60 +lat_0=42.5 +lon_0=-100 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
crs_geo_str = '+proj=longlat +datum=WGS84 +no_defs'
crs_from = CRS.from_proj4(crs_geo_str)
crs_to = CRS.from_proj4(crs_pro_str)
new_shps = gpd.read_file(shp_file)
polygon = new_shps.at[0, 'geometry']
start = time.time()
mask = create_mask(polygon, x, y, lons, lats, crs_from, crs_to)
end = time.time()
print('time：', '%.7f' % (end - start))
hydro_util.serialize_numpy(np.array(mask), mask_file)
var_types = ['tmax']
# var_types = ['tmax', 'tmin', 'prcp', 'srad', 'vp', 'swe', 'dayl']
avgs = []
for var_type in var_types:
start = time.time()
avg = calc_avg(mask, data_netcdf, var_type)
end = time.time()
print('time：', '%.7f' % (end - start))
print('mean value：', avg)
avgs.append(avg)
return avgs
def _create_nc_file_time1d(self,
a,
fname,
varname,
varunits=None,
fill=None):
self.log.debug('a={0}'.format(a))
self.log.debug('shape a = {0}'.format(a.shape))
assert len(a.shape) == 1
time_dim = len(a)
rootgrp = Dataset(fname, "w", format="NETCDF4")
rootgrp.createDimension("time", time_dim)
vals = rootgrp.createVariable(varname,
"f4",
dimensions=("time", ),
fill_value=fill)
times = rootgrp.createVariable("time", "f4", dimensions=("time", ))
vals[:] = [d['mean'] for d in a]
times[:] = [d['time'] for d in a]
if varunits is not None:
vals.units = varunits
times.units = 'seconds since 1970-01-01 00:00:00'
rootgrp.close()
class NetCDFConverter(object):
def __init__(self, metadataFile, csvFile, ncOutput):
Log().set_log_info("[Begin] conversion to NetCDF of: " + metadataFile +
" " + csvFile + " " + ncOutput)
self.init_elements(metadataFile, csvFile, ncOutput)
if not os.path.exists(self.ncOutput):
try:
self.ncFile = Dataset(self.ncOutput,
'w',
format='NETCDF' + self.version)
except:
Log().set_log_error(
"The netCDF_version is wrong. Assigning the default value(netCDF4_CLASSIC) to "
"netCDF_version")
self.version = '4_CLASSIC'
self.ncFile = Dataset(self.ncOutput,
'w',
format='NETCDF' + self.version)
self.create_netcdf()
else: # Append start
self.ncFile = Dataset(self.ncOutput, 'r+')
self.append_netcdf()
self.metadata.globalAttributes.max_min_attribute(self.ncFile)
self.ncFile.close()
Log().set_log_info("[Finished] conversion to NetCDF of : " +
metadataFile + " " + csvFile + " " + ncOutput)
def init_elements(self, metadataFile, csvFile, ncOutput):
self.ncOutput = ncOutput
self.ncOutput = Checker().check_source(metadataFile, csvFile, ncOutput)
self.metadata = Metadata(metadataFile)
self.metadataData = self.metadata.get_metadata()
self.data = Data(csvFile)
self.ncOutput = self.ncOutput + self.metadata.get_global_attributes(
).get_id() + ".nc"
self.version = self.metadata.get_global_attributes(
).get_netcdf_version()
self.temporalAppendPosition = {}
self.globalAttributes = Metadata(metadataFile).get_global_attributes()
self.dimensions = self.metadata.get_dimensions()
self.naming_authority = self.globalAttributes.attributesList[
'naming_authority']
if self.naming_authority == 'EGO':
self.data_ego = Data_ego(csvFile)
self.ego_standard_metadata = EgoReaderStandardMetadata()
self.dimensionsEgo = self.ego_standard_metadata.get_dimensions()
self.metadata.change_variable()
def create_netcdf(self):
self.globalAttributes.write_attributes(self.ncFile)
self.dimensions.write_dimensions(self.ncFile)
if self.naming_authority == 'EGO':
self.dimensionsEgo.write_dimensions(self.ncFile)
self.variables = self.ego_standard_metadata.get_glider_characteristics_variables(
)
writer = writer_ego_standard(
self.dimensions.get_metadata_dimension())
writer.write(self.variables,
self.metadataData['glider_characteristics'],
self.ncFile)
#self.writer.write_variables_data(self.metadataData['glider_characteristics'], self.variables, self.version)
self.variables = self.ego_standard_metadata.get_glider_deployment_variables(
)
writer = writer_ego_standard(
self.dimensions.get_metadata_dimension())
writer.write(self.variables,
self.metadataData['glider_deployment'], self.ncFile)
self.variables = self.metadata.get_variables()
self.writer_ego = writer_ego(self.data_ego, self.dimensions,
self.ncFile)
self.writer_ego.write_variables_data(
self.metadataData['variables'], self.variables, self.version)
else:
self.variables = self.metadata.get_variables()
self.writer = Writer(self.data, self.dimensions, self.ncFile)
self.writer.write_variables_data(self.metadataData['variables'],
self.variables, self.version)
def append_netcdf(self):
self.dimensions.set_dimensions_by_netcdf(self.ncFile.dimensions)
self.writer = Writer(self.data, self.dimensions, self.ncFile)
self.writer.write_append_variables_data(self.metadata.get_variables())
def insert_variable(self,
ncfile,
var_name,
domain_key=None,
replace_data=False,
sql_where="true",
file_type='windb2',
mask=None,
zero_seconds=False):
"""Inserts a netCDF file with WinDB2 or WRF output into a WinDB2 database.
*
* windb2Conn - Connection to a WinDB2 database.
* ncfile - Either an open file or a string name of a file to open.
* var_name - Name of WinDB2 supported variable or a WRF 3D variable (currently WIND, THETA, RHO).
* domain_key - Existing domain key in the database. If left blank, a new domain will be created.
* replace_data - Deletes data for the same time in the database if True. Useful for freshening data.
* file_type - Type of netCDF file to insert: {'windb2' (default), or 'wrf'}
* mask - String name of a mask in the WinDB2 database. Only relevant when creating a new domain (the mask is
* applied automatically thereafter).
*
* returns timesInsertedList, domain_key - A list of times inserted in ISO time format, and the
domain_key where the data was inserted.
:param file_type:
"""
# Make sure this the file_type of file is support
if file_type != 'windb2' and file_type != 'wrf':
raise TypeError('Unsupported file file_type: {}'.format(file_type))
# Open the WinDB netCDF file
logger.debug(
'netCDF file file_type passed to wrf.insertNcFile={}'.format(
type(ncfile)))
if type(ncfile) != Dataset:
ncfile = Dataset(ncfile, 'r')
# Get the grid dimensions and coordinates
if file_type == 'windb2':
nlong = len(ncfile.dimensions['x'])
nlat = len(ncfile.dimensions['y'])
x_coord_array = ncfile.groups['WRF']['XLONG'][:]
y_coord_array = ncfile.groups['WRF']['XLAT'][:]
if self.config['vars'][var_name]['dims'] == 3:
height_array = ncfile.variables['height'][:]
elif self.config['vars'][var_name]['dims'] == 2:
height_array = [self.config['vars'][var_name]['insert'][0]]
else:
raise Exception('Number of dimensions must either be 2 or 3')
init_t = datetime.strptime(
ncfile.groups['WRF'].SIMULATION_START_DATE,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
elif file_type == 'wrf':
nlong = len(ncfile.dimensions['west_east'])
nlat = len(ncfile.dimensions['south_north'])
x_coord_array = ncfile['XLONG'][:]
y_coord_array = ncfile['XLAT'][:]
height_array = [self.config[var_name]['insert'][0]]
init_t = datetime.strptime(
ncfile.SIMULATION_START_DATE,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
# Read in the vars to insert
wrf_copied_var = False
if file_type == 'windb2' and var_name.lower() == 'WIND'.lower():
u = ncfile.variables['eastward_wind'][:]
v = ncfile.variables['northward_wind'][:]
elif file_type == 'windb2' and var_name.lower() == 'DPT'.lower():
ncVariable = ncfile.variables['dew_point_temperature'][:]
# Otherwise try find the WinDB2 interp or WRF var
else:
try:
ncVariable = ncfile.variables[var_name][:]
except KeyError as e:
wrf_copied_var = True
ncVariable = ncfile.groups['WRF'][var_name][:]
# Create a new and/or domain if necessary
if domain_key is None:
if file_type == 'windb2':
domain_key = str(
self.create_new_domain(ncfile.groups['WRF'].TITLE, "WRF",
ncfile.groups['WRF'].DX, 'm', mask))
elif file_type == 'wrf':
domain_key = str(
self.create_new_domain(ncfile.TITLE, "WRF", ncfile.DX, 'm',
mask))
self.insert_horiz_geom(domain_key, x_coord_array, y_coord_array,
create_wrf_srid(self.windb2, ncfile))
# Mask the domain if necessary
if mask is not None:
self.mask_domain(domain_key, mask)
# Create a new table if necessary and add an initialization time column
if file_type == 'windb2' and var_name.lower() == 'wind'.lower():
if not self.windb2.table_exists('wind' + '_' + domain_key):
self.create_new_table(domain_key, var_name,
('speed', 'direction'),
('real', 'smallint'))
self._create_initialization_time_column(var_name, domain_key)
else:
if not self.windb2.table_exists('{}_{}'.format(
var_name.lower(), domain_key)):
self.create_new_table(domain_key, var_name, ('value', ),
('real', ))
self._create_initialization_time_column(
var_name.lower(), domain_key)
# Make sure it's a string so that we don't have concatenation problems later
domain_key = str(domain_key)
# Get the geomkeys associated with the WRF coordinates
horizGeomKey = self.calculateHorizWindGeomKeys(domain_key, nlong, nlat)
# Create a counter to execute every so often
counter = 0
startTime = datetime.now()
# Get the time array to iterate through
if file_type == 'windb2':
time_char_array = chartostring(ncfile.variables['Time'][:])
elif file_type == 'wrf':
time_char_array = chartostring(ncfile.variables['Times'][:])
# Create a statement to use
tCount = 0
timeValuesToReturn = []
for t in time_char_array:
# Create a datetime from the WRF string
t = datetime.strptime(t,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
# Zero the seconds if asked to
if zero_seconds:
t = t.replace(second=0)
# Create the time in GeoServer/GeoWebCache format
timeValuesToReturn.append(t.strftime('%Y-%m-%dT%H:%M:%S.000Z'))
# Info
print('Processing time for {}: {}'.format(var_name,
timeValuesToReturn[-1]))
# Iterate through the x,y, and timearr and insert the WRF variable
for h in height_array:
# We actually need the index of the height, not the actual height itself
height = None
if file_type == 'windb2' and wrf_copied_var is False:
try:
z = numpy.argwhere(
ncfile.variables['height'][:] == h)[0, 0]
height = height_array[z]
except IndexError:
logger.error(
'Height {} to insert does not exist in WinDB2 file'
.format(h))
sys.exit(-1)
elif wrf_copied_var is True:
height = height_array[0]
else:
height = 0
counter = 0
# Open a temporary file to COPY from
tempFile = tempfile.NamedTemporaryFile(mode='w')
for x in range(horizGeomKey.shape[0]):
for y in range(horizGeomKey.shape[1]):
# Make sure that this is actually a x,y point we want to insert
# In order to create a mask of selective insert points, all
# a horizGeomKey of zero means we don't want to insert this one
if horizGeomKey[x, y] == 0:
continue
# Write the data string to the temp file
if file_type == 'windb2' and var_name.lower(
) == 'wind'.lower():
if not (numpy.isnan(u[tCount, z, y, x])
or numpy.isnan(v[tCount, z, y, x])):
# Add this row to be inserted into the database
# Note that we negate U and V so they exist in WinDB2 as the vernacular "coming from" wind direction
print('{}, {}, {}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'),
util.speed(u[tCount, z, y, x], v[tCount, z,
y, x]),
int(
util.calc_dir_deg(
-u[tCount, z, y, x],
-v[tCount, z, y, x])), height,
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
elif file_type == 'windb2':
# Add this row to be inserted into the database
if self.config['vars'][var_name]['dims'] == 2:
val = ncVariable[tCount, y, x]
elif self.config['vars'][var_name]['dims'] == 3:
val = ncVariable[tCount, z, y, x]
if not numpy.isnan(val):
print('{}, {}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'), val,
height,
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
elif file_type == 'wrf':
if not numpy.isnan(ncVariable[tCount, y, x]):
# Add this row to be inserted into the database
print('{}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'),
ncVariable[tCount, y, x],
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
# Insert the data
tempFile.flush()
if file_type == 'windb2' and var_name.lower() == 'wind'.lower(
):
insertColumns = columns = ('domainkey', 'geomkey', 't',
'speed', 'direction', 'height',
'init')
else:
insertColumns = columns = ('domainkey', 'geomkey', 't',
'value', 'height', 'init')
try:
self.windb2.curs.copy_from(open(tempFile.name, 'r'),
var_name + '_' + domain_key,
sep=',',
columns=insertColumns)
except psycopg2.IntegrityError as e:
# Delete the duplicate data
errorTest = 'duplicate key value violates unique constraint "' + var_name.lower(
) + "_" + domain_key + '_domainkey_geomkey_t_height_init_key"'
if re.search(errorTest, str(e.pgerror)):
# Delete the data and retry the insert if asked to replace data in the function call
if replace_data:
# Rollback to the last commit (necessary to reset the database connection)
self.windb2.conn.rollback()
# Delete that timearr (assumes UTC timearr zone)
sql = 'DELETE FROM ' + var_name + '_' + domain_key + \
' WHERE t = timestamp with time zone\'' + t.strftime('%Y-%m-%d %H:%M:%S %Z') + '\' ' + \
'AND height=' + str(h)
print("Deleting conflicting times: " + sql)
self.windb2.curs.execute(sql)
self.windb2.conn.commit()
# Reinsert that timearr
self.windb2.curs.copy_from(
open(tempFile.name, 'r'),
var_name + '_' + domain_key,
sep=',',
columns=insertColumns)
# Commit again or the reinserts won't stick
self.windb2.conn.commit()
continue
# Otherwise, just notify that the insert failed because of duplicate data. We do re-raise this error
# because it's assumed that we want to suplement the WinDB with other data-heights if available.
else:
logging.warning('ERROR ON INSERT: {}'.format(
e.pgerror))
logging.warning(
'Use \'replace_data=True\' if you want the data to be reinserted.'
)
self.windb2.conn.rollback()
continue
# Commit the changes
self.windb2.conn.commit()
# Calaculate the insert rate
elapsedTime = (datetime.now() - startTime).seconds
try:
print('Inserted {}, {}-m height x,y wind points at {} I/s'.
format(counter, height_array[z],
counter / elapsedTime))
except ZeroDivisionError:
print('Inserted {}, {}-m height x,y wind points'.format(
counter, height_array[z]))
except UnboundLocalError:
print('Inserted {}, {}-m height x,y wind points'.format(
counter, height_array[0]))
# Close the tempfile so it is deleted
tempFile.close()
# Increment the time
tCount += 1
return timeValuesToReturn, domain_key
def build_image(self, listimage, output_images, region, verbose):
if verbose:
print("[I] - Building just one image.")
#Variables to create netcdf file
resolution=0.0025
lonbegin=get_coordinates(region)['upper_left'][0]
latbegin = get_coordinates(region)['upper_left'][1]
if len(listimage)==1:
# Open matrix 1 and 2 from tif images to create newmatrix
imga = rasterio.open(listimage[0]).read(1)
newmatrix = imga
# close all files used
del imga
elif len(listimage)==2:
#Open matrix 1 and 2 from tif images to create newmatrix
imga = rasterio.open(listimage[0]).read(1)
imgb = rasterio.open(listimage[1]).read(1)
newmatrix = numpy.maximum(imga, imgb)
# close all files used
del imga
del imgb
elif len(listimage)>2:
return
#Defined number of column and line over the bigger matrix
#NC = abs(round(((lonend - lonbegin) / resolution) + 1))
#NL = abs(round(((latend - latbegin) / resolution) + 1))
NC = newmatrix.shape[0]
NL = newmatrix.shape[1]
#Path where will be save the new netcdf
#namenewfile = output_images+listimage[0].replace(listimage[0].split(".")[-3], date).replace(listimage[0].split(".")[-1],
namenewfile = output_images+listimage[0].split('/')[-1].replace('.tif', '_'+listimage[0].split('/')[-2].split('_')[-1])+".nc"
#New netcdf file
newfile = Dataset(namenewfile, 'w', format='NETCDF4')
#Create vetor of latitude and logitude
new_lat = [latbegin + i *resolution for i in range(0,NL)]
new_lon = [lonbegin + i *resolution for i in range(0,NC)][::-1]
#Defined Dimension of netcdf file
newfile.createDimension("lat", len(new_lat))
newfile.createDimension("lon", len(new_lon))
#Defined variables of netcdf file
latitude = newfile.createVariable("lat","f4",("lat",))
longitude = newfile.createVariable("lon","f4",("lon",))
value = newfile.createVariable("value","f4",("lon","lat",))
#fill netcdf's variables with newmatrix, latitude and logitude vetors
value[:,:]=newmatrix
latitude[:]=new_lat
longitude[:]=new_lon
newfile.close()
print("Builded file: ", namenewfile)
#Delete newmatrix
del newmatrix
print('OS error occurs, a directory {} was not created'.format(
save_path))
raise OSError
with open(save_path + LOG, 'w') as g:
g.write('Current dir: ' + nc_path + '\n\n')
g.write('Coordinate range: ' + output_dir_name + '\n\n')
# If the file is open for write access (mode='w', 'r+' or 'a'), you may write any type of data including new
# dimensions, groups, variables and attributes.
# data_model= NETCDF3_CLASSIC
file_count = 0
for i in range(len(file_names)):
prof_numbers_to_save = []
dataset = Dataset(nc_path + file_names[i],
"r",
format="NETCDF3_CLASSIC")
n_prof = dataset.dimensions['N_PROF'].size # число профилей
latitude = dataset.variables['LATITUDE'] # массив 1 х n_prof из широты
longitude = dataset.variables[
'LONGITUDE'] # массив 1 х n_prof из широты
with open(save_path + LOG, 'a') as g:
g.write(file_names[i] + '\n')
g.write('Coordinate range: Lat: {}, {}; Long: {}, {} - '.format(
min(latitude), max(latitude), min(longitude), max(longitude)))
for j in range(n_prof):
if (lat_edge[0] <= latitude[j] <= lat_edge[1]) and (
long_edge[0] <= longitude[j] <= long_edge[1]):
prof_numbers_to_save.append(j)
if len(prof_numbers_to_save) == 0:
file_names = []
if os.path.isfile(nc_path):
print(nc_path, ' - файл')
elif os.path.isdir(nc_path):
for item in os.listdir(nc_path):
if os.path.isfile(nc_path + item):
fn_pattern = r"filtered_[\d]{8}_prof_*[\d]*\.nc"
parce = re.search(fn_pattern, item)
if parce:
file_names.append(nc_path + item)
profiles = []
indices = []
count = 0
for fn in file_names:
print(fn)
dataset = Dataset(fn, "r", format="NETCDF3_CLASSIC")
n_prof = dataset.dimensions['N_PROF'].size # profile number
n_levels = dataset.dimensions['N_LEVELS'].size # profile number
juld_location = get_jul_dates(dataset)
latitude = dataset.variables['LATITUDE']
salinity = dataset.variables['PSAL_ADJUSTED']
salinity_qc = dataset.variables['PSAL_ADJUSTED_QC']
salinity_err = dataset.variables['PSAL_ADJUSTED_ERROR']
pressure = dataset.variables['PRES_ADJUSTED']
pressure_qc = dataset.variables['PRES_ADJUSTED_QC']
pressure_err = dataset.variables['PRES_ADJUSTED_ERROR']
temperature = dataset.variables['TEMP_ADJUSTED']
temperature_qc = dataset.variables['TEMP_ADJUSTED_QC']
temperature_err = dataset.variables['TEMP_ADJUSTED_ERROR']
for i in range(n_prof):