def testCalcDirDegrees(self):
# First quadrant
self.assertEqual(util.calc_dir_deg(3, 3), 45)
# Second quadrant
self.assertEqual(util.calc_dir_deg(3, -3), 135)
# Third quadrant
self.assertEqual(util.calc_dir_deg(-3, -3), 225)
# Fourth quadrant
self.assertEqual(util.calc_dir_deg(-3, 3), 315)
def uv_column_interp(u_mass_column, v_mass_column, heights_above_ground, heights_to_interp, z_o=None):
"""Calculate the wind speeds at a given height (in meters) using the two adjacent pressure levels u and v
wind components. Uses a log-law approximation from the ground to 100 m for points that are lower than the
lowest eta level in the model. Direction interpolation is not performed for levels where the log-law
approximation is used.
:param u_mass_column: WRF U wind speed already interpolated to the mass coordinate system (native is flux coordinate)
:param v_mass_column: WRF V wind speed already interpolated to the mass coordinate system (native is flux coordinate)
:param heights_above_ground:
:param heights_to_interp:
:returns 2, 1D arrays u, v of the height interpolated wind speed and direction in the column.
"""
# Make sure array lengths match
assert(u_mass_column.shape == v_mass_column.shape)
assert(u_mass_column.shape == heights_above_ground.shape)
# Make sure we've only been given a column
assert(u_mass_column.ndim == 1)
assert(v_mass_column.ndim == 1)
assert(heights_above_ground.ndim == 1)
# Convert heights_to_interp to an numpy array so we can mask it
heights_to_interp = numpy.array(heights_to_interp)
# Find the heights that need log-law interpolation i.e. below 100 m and lower than the lowest eta-half level
mask_heights_to_interp_log_law = heights_to_interp < numpy.min(heights_above_ground)
# Linear interp where we have model data
if numpy.size(heights_to_interp[~mask_heights_to_interp_log_law]) > 0:
u_mass_coord_interp_linear = numpy.interp(heights_to_interp[~mask_heights_to_interp_log_law],
heights_above_ground, u_mass_column)
v_mass_coord_interp_linear = numpy.interp(heights_to_interp[~mask_heights_to_interp_log_law],
heights_above_ground, v_mass_column)
else:
logger.info('No eta-level heights were in the range of LINEAR interpolation.')
u_mass_coord_interp_linear = []
v_mass_coord_interp_linear = []
# Log-law interp for speed using heights 100 m and below, if WRF ZNT was NOT provided
speed = util.speed(u_mass_column, v_mass_column)
if numpy.size(heights_to_interp[mask_heights_to_interp_log_law]) > 0 and z_o is None:
# Make sure there is more than one height to linearly interpolate with below 100 m
if heights_above_ground[1] < 100:
speed_interp_log_law = log_law_interp(speed, heights_above_ground,
heights_to_interp[mask_heights_to_interp_log_law], 100)
# Otherwise, issue a warning and include the second height above ground
# This situation can sometimes occur at the edges of nested domains when the PSFC value
# is lower than the lowest level of (P + PB). Speculate this is a bug in WRF where the PSFC is being taken
# from the lower resolution domain rather than being calculated with the new topo height in the higher
# resolution domain.
else:
speed_interp_log_law = log_law_interp(speed, heights_above_ground,
heights_to_interp[mask_heights_to_interp_log_law], heights_above_ground[1])
logger.info('Had to use height above 100 m for log-linear log-law interp')
logger.info('Heights used for log-linear log-law interp: {} m and {} m'
.format(heights_above_ground[0], heights_above_ground[1]))
# Use the log-law with the WRF ZNT (surface roughness) to diagnose U,V
elif numpy.size(heights_to_interp[mask_heights_to_interp_log_law]) > 0 and z_o is not None:
speed_interp_log_law = speed[0] * \
numpy.log(heights_to_interp[mask_heights_to_interp_log_law] / z_o) / \
numpy.log(numpy.min(heights_above_ground) / z_o)
else:
logger.info('No eta-level heights were in the range of LOG-LAW interpolation.')
speed_interp_log_law = []
# For log-law interpolation, use the same wind direction as the bottom level where we have WRF data for the
# direction. Assumes these heights are sorted from bottom to top, which is the WRF default.
u_bottom = u_mass_column[0]
v_bottom = v_mass_column[0]
dir_interp_log_law = util.calc_dir_deg(u_bottom, v_bottom)
u_mass_coord_interp_log_law = util.u_flow(speed_interp_log_law, dir_interp_log_law)
v_mass_coord_interp_log_law = util.v_flow(speed_interp_log_law, dir_interp_log_law)
return numpy.hstack((u_mass_coord_interp_log_law, u_mass_coord_interp_linear)), \
numpy.hstack((v_mass_coord_interp_log_law, v_mass_coord_interp_linear))
def insertNcFile(windb2_conn, ncFile, domainKey=None, tableName="current", replaceData=False, sqlWhere="true"):
# Connect to the WinDB
inserter = insert.Insert(windb2_conn)
# Open the tide netCDF file
print('netCDF file type passed to suntans.insertNcFile=', type(ncFile))
if type(ncFile) != nc.netcdf_file:
ncFile = nc.netcdf_file(ncFile, 'r')
# Get the grid dimensions and coordinates
nLong = ncFile.dimensions['west_east']
nLat = ncFile.dimensions['south_north']
nTime = ncFile.variables['Times'].shape[0] # 'Time' dim is UNLIMITED
lonArr = ncFile.variables['utm_easting']
latArr = ncFile.variables['utm_northing']
timeArr = ncFile.variables['Times']
u = ncFile.variables['u_top1m']
v = ncFile.variables['v_top1m']
# Create a new domain if necessary
if domainKey == None:
domainKey = str(inserter.create_new_domain("SF Bay currents ", "SUNTANS", 200, 'm'))
inserter.insert_horiz_geom(domainKey, lonArr, latArr, 26910) # EPSG:26910: NAD83 / UTM zone 10N
inserter.create_new_table(domainKey, tableName, ("speed", "direction"), ("real", "smallint"),
("speed_positive","direction_degree"), ("speed >= 0","direction >= 0 AND direction < 360"))
else:
# Make sure it's a string so that we don't have concatenation problems later
domainKey = str(domainKey)
# Get the geomkeys associated with the WRF coordinates
horizGeomKey = inserter.calculateHorizWindGeomKeys(domainKey, nLong, nLat)
# Create a counter to execute every so often
counter = 0
startTime = datetime.now()
# Pass the timearr through the timearr filter, even if no filter is set to return Postres
# compliant timestamps. See Python datetime.datetime for datetime format details
timesToInsert = windb2_conn.filterTimes(timeArr, '%Y-%m-%d_%H:%M:%S', sqlWhere=sqlWhere)
# Info
print('Reduced the number of times to insert by ',
round((1 - float(len(timesToInsert)) / timeArr.shape[0]) * 100, 1), '%')
# Iterate through the times that we want to insert
ttiCount = 0
tncfCount = 0
timeValuesToReturn = []
while ttiCount < len(timesToInsert) and tncfCount < nTime:
# Only insert if this is a time we want to insert
tncf = datetime.strptime(timeArr[tncfCount].tostring().decode('UTF-8'), '%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
tti = timesToInsert[ttiCount]
if tti != tncf:
tncfCount += 1
continue
# Open a temporary file to COPY from
temp_file = tempfile.NamedTemporaryFile(mode='w+b')
# Create the time in GeoServer/GeoWebCache format
timeValuesToReturn.append(tncf.strftime('%Y-%m-%dT%H:%M:%S.000Z'))
# Info
print('Processing time: ', timeValuesToReturn[-1])
# Iterate through the x,y, and timearr and insert the tidal current data
for x in range(horizGeomKey.shape[0]):
for y in range(horizGeomKey.shape[1]):
# Make sure that this is actually a x,y point we want to insert
# In order to create a mask of selective insert points, all
# a horizGeomKey of zero means we don't want to insert this one
if horizGeomKey[x,y]==0:
continue;
# Write the data string to the temp file
if not numpy.isnan(u[tncfCount,y,x]):
# Calculate speed
speed = math.sqrt(math.pow(u[tncfCount,y,x],2) + math.pow(v[tncfCount,y,x],2))
# Calculate direction (using the 'flow' convention for tides)
dir = int(util.calc_dir_deg(u[tncfCount,y,x], v[tncfCount,y,x]))
# Add this row to be inserted into the database
str_to_write = '{},{},{},{},{},{}\n'.format(domainKey, horizGeomKey[x,y],
tncf.strftime('%Y-%m-%d %H:%M:%S %Z'), speed, dir, 0)
temp_file.write(bytes(str_to_write))
counter += 1
# Insert the data at height 0 for tidal current
temp_file.flush()
insertColumns = ('domainkey', 'geomkey', 't', 'speed', 'direction', 'height')
try:
windb2_conn.curs.copy_from(open(temp_file.name, 'rb'), tableName + '_' + domainKey, sep=',', columns=insertColumns)
except psycopg2.IntegrityError as e:
# Delete the duplicate data
errorTest = 'duplicate key value violates unique constraint "' + tableName + "_" + domainKey + '_domainkey_geomkey_t_height_key"'
if re.search(errorTest, str(e)):
# Delete the data and retry the insert if asked to replace data in the function call
if replaceData:
# Rollback to the last commit (necessary to reset the database connection)
windb2_conn.conn.rollback()
# Delete that timearr (assumes UTC timearr zone)
sql = 'DELETE FROM ' + tableName + '_' + domainKey + ' WHERE t = timestamp with time zone\'' + \
tti.strftime('%Y-%m-%d %H:%M:%S %Z') + '\' ' + \
'AND height=0'
print("Deleting conflicting times: " + sql)
windb2_conn.curs.execute(sql)
windb2_conn.conn.commit()
# Reinsert that timearr
windb2_conn.curs.copy_from(open(temp_file.name, 'r'), tableName + '_' + domainKey, sep=',', columns=insertColumns)
# Otherwise, just notify that the insert failed because of duplicate data
else:
logging.error("ERROR ON INSERT: ", e.message)
logging.error("Use 'replaceData=True' if you want the data to be reinserted.")
raise
# Commit the changes
windb2_conn.conn.commit()
# Calaculate the insert rate
elapsedTime = (datetime.now() - startTime).seconds
if elapsedTime > 0:
insertRate = counter / elapsedTime
print("Inserted ", counter, " x,y wind points at ", insertRate, " I/s")
# Close the tempfile so it is deleted
temp_file.close()
# Increment the time
ttiCount += 1
return timeValuesToReturn
def insert_variable(self,
ncfile,
var_name,
domain_key=None,
replace_data=False,
sql_where="true",
file_type='windb2',
mask=None,
zero_seconds=False):
"""Inserts a netCDF file with WinDB2 or WRF output into a WinDB2 database.
*
* windb2Conn - Connection to a WinDB2 database.
* ncfile - Either an open file or a string name of a file to open.
* var_name - Name of WinDB2 supported variable or a WRF 3D variable (currently WIND, THETA, RHO).
* domain_key - Existing domain key in the database. If left blank, a new domain will be created.
* replace_data - Deletes data for the same time in the database if True. Useful for freshening data.
* file_type - Type of netCDF file to insert: {'windb2' (default), or 'wrf'}
* mask - String name of a mask in the WinDB2 database. Only relevant when creating a new domain (the mask is
* applied automatically thereafter).
*
* returns timesInsertedList, domain_key - A list of times inserted in ISO time format, and the
domain_key where the data was inserted.
:param file_type:
"""
# Make sure this the file_type of file is support
if file_type != 'windb2' and file_type != 'wrf':
raise TypeError('Unsupported file file_type: {}'.format(file_type))
# Open the WinDB netCDF file
logger.debug(
'netCDF file file_type passed to wrf.insertNcFile={}'.format(
type(ncfile)))
if type(ncfile) != Dataset:
ncfile = Dataset(ncfile, 'r')
# Get the grid dimensions and coordinates
if file_type == 'windb2':
nlong = len(ncfile.dimensions['x'])
nlat = len(ncfile.dimensions['y'])
x_coord_array = ncfile.groups['WRF']['XLONG'][:]
y_coord_array = ncfile.groups['WRF']['XLAT'][:]
if self.config['vars'][var_name]['dims'] == 3:
height_array = ncfile.variables['height'][:]
elif self.config['vars'][var_name]['dims'] == 2:
height_array = [self.config['vars'][var_name]['insert'][0]]
else:
raise Exception('Number of dimensions must either be 2 or 3')
init_t = datetime.strptime(
ncfile.groups['WRF'].SIMULATION_START_DATE,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
elif file_type == 'wrf':
nlong = len(ncfile.dimensions['west_east'])
nlat = len(ncfile.dimensions['south_north'])
x_coord_array = ncfile['XLONG'][:]
y_coord_array = ncfile['XLAT'][:]
height_array = [self.config[var_name]['insert'][0]]
init_t = datetime.strptime(
ncfile.SIMULATION_START_DATE,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
# Read in the vars to insert
wrf_copied_var = False
if file_type == 'windb2' and var_name.lower() == 'WIND'.lower():
u = ncfile.variables['eastward_wind'][:]
v = ncfile.variables['northward_wind'][:]
elif file_type == 'windb2' and var_name.lower() == 'DPT'.lower():
ncVariable = ncfile.variables['dew_point_temperature'][:]
# Otherwise try find the WinDB2 interp or WRF var
else:
try:
ncVariable = ncfile.variables[var_name][:]
except KeyError as e:
wrf_copied_var = True
ncVariable = ncfile.groups['WRF'][var_name][:]
# Create a new and/or domain if necessary
if domain_key is None:
if file_type == 'windb2':
domain_key = str(
self.create_new_domain(ncfile.groups['WRF'].TITLE, "WRF",
ncfile.groups['WRF'].DX, 'm', mask))
elif file_type == 'wrf':
domain_key = str(
self.create_new_domain(ncfile.TITLE, "WRF", ncfile.DX, 'm',
mask))
self.insert_horiz_geom(domain_key, x_coord_array, y_coord_array,
create_wrf_srid(self.windb2, ncfile))
# Mask the domain if necessary
if mask is not None:
self.mask_domain(domain_key, mask)
# Create a new table if necessary and add an initialization time column
if file_type == 'windb2' and var_name.lower() == 'wind'.lower():
if not self.windb2.table_exists('wind' + '_' + domain_key):
self.create_new_table(domain_key, var_name,
('speed', 'direction'),
('real', 'smallint'))
self._create_initialization_time_column(var_name, domain_key)
else:
if not self.windb2.table_exists('{}_{}'.format(
var_name.lower(), domain_key)):
self.create_new_table(domain_key, var_name, ('value', ),
('real', ))
self._create_initialization_time_column(
var_name.lower(), domain_key)
# Make sure it's a string so that we don't have concatenation problems later
domain_key = str(domain_key)
# Get the geomkeys associated with the WRF coordinates
horizGeomKey = self.calculateHorizWindGeomKeys(domain_key, nlong, nlat)
# Create a counter to execute every so often
counter = 0
startTime = datetime.now()
# Get the time array to iterate through
if file_type == 'windb2':
time_char_array = chartostring(ncfile.variables['Time'][:])
elif file_type == 'wrf':
time_char_array = chartostring(ncfile.variables['Times'][:])
# Create a statement to use
tCount = 0
timeValuesToReturn = []
for t in time_char_array:
# Create a datetime from the WRF string
t = datetime.strptime(t,
'%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
# Zero the seconds if asked to
if zero_seconds:
t = t.replace(second=0)
# Create the time in GeoServer/GeoWebCache format
timeValuesToReturn.append(t.strftime('%Y-%m-%dT%H:%M:%S.000Z'))
# Info
print('Processing time for {}: {}'.format(var_name,
timeValuesToReturn[-1]))
# Iterate through the x,y, and timearr and insert the WRF variable
for h in height_array:
# We actually need the index of the height, not the actual height itself
height = None
if file_type == 'windb2' and wrf_copied_var is False:
try:
z = numpy.argwhere(
ncfile.variables['height'][:] == h)[0, 0]
height = height_array[z]
except IndexError:
logger.error(
'Height {} to insert does not exist in WinDB2 file'
.format(h))
sys.exit(-1)
elif wrf_copied_var is True:
height = height_array[0]
else:
height = 0
counter = 0
# Open a temporary file to COPY from
tempFile = tempfile.NamedTemporaryFile(mode='w')
for x in range(horizGeomKey.shape[0]):
for y in range(horizGeomKey.shape[1]):
# Make sure that this is actually a x,y point we want to insert
# In order to create a mask of selective insert points, all
# a horizGeomKey of zero means we don't want to insert this one
if horizGeomKey[x, y] == 0:
continue
# Write the data string to the temp file
if file_type == 'windb2' and var_name.lower(
) == 'wind'.lower():
if not (numpy.isnan(u[tCount, z, y, x])
or numpy.isnan(v[tCount, z, y, x])):
# Add this row to be inserted into the database
# Note that we negate U and V so they exist in WinDB2 as the vernacular "coming from" wind direction
print('{}, {}, {}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'),
util.speed(u[tCount, z, y, x], v[tCount, z,
y, x]),
int(
util.calc_dir_deg(
-u[tCount, z, y, x],
-v[tCount, z, y, x])), height,
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
elif file_type == 'windb2':
# Add this row to be inserted into the database
if self.config['vars'][var_name]['dims'] == 2:
val = ncVariable[tCount, y, x]
elif self.config['vars'][var_name]['dims'] == 3:
val = ncVariable[tCount, z, y, x]
if not numpy.isnan(val):
print('{}, {}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'), val,
height,
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
elif file_type == 'wrf':
if not numpy.isnan(ncVariable[tCount, y, x]):
# Add this row to be inserted into the database
print('{}, {}, {}, {}, {}'.format(
domain_key, horizGeomKey[x, y],
t.strftime('%Y-%m-%d %H:%M:%S %Z'),
ncVariable[tCount, y, x],
init_t.strftime('%Y-%m-%d %H:%M:%S %Z')),
file=tempFile)
counter += 1
# Insert the data
tempFile.flush()
if file_type == 'windb2' and var_name.lower() == 'wind'.lower(
):
insertColumns = columns = ('domainkey', 'geomkey', 't',
'speed', 'direction', 'height',
'init')
else:
insertColumns = columns = ('domainkey', 'geomkey', 't',
'value', 'height', 'init')
try:
self.windb2.curs.copy_from(open(tempFile.name, 'r'),
var_name + '_' + domain_key,
sep=',',
columns=insertColumns)
except psycopg2.IntegrityError as e:
# Delete the duplicate data
errorTest = 'duplicate key value violates unique constraint "' + var_name.lower(
) + "_" + domain_key + '_domainkey_geomkey_t_height_init_key"'
if re.search(errorTest, str(e.pgerror)):
# Delete the data and retry the insert if asked to replace data in the function call
if replace_data:
# Rollback to the last commit (necessary to reset the database connection)
self.windb2.conn.rollback()
# Delete that timearr (assumes UTC timearr zone)
sql = 'DELETE FROM ' + var_name + '_' + domain_key + \
' WHERE t = timestamp with time zone\'' + t.strftime('%Y-%m-%d %H:%M:%S %Z') + '\' ' + \
'AND height=' + str(h)
print("Deleting conflicting times: " + sql)
self.windb2.curs.execute(sql)
self.windb2.conn.commit()
# Reinsert that timearr
self.windb2.curs.copy_from(
open(tempFile.name, 'r'),
var_name + '_' + domain_key,
sep=',',
columns=insertColumns)
# Commit again or the reinserts won't stick
self.windb2.conn.commit()
continue
# Otherwise, just notify that the insert failed because of duplicate data. We do re-raise this error
# because it's assumed that we want to suplement the WinDB with other data-heights if available.
else:
logging.warning('ERROR ON INSERT: {}'.format(
e.pgerror))
logging.warning(
'Use \'replace_data=True\' if you want the data to be reinserted.'
)
self.windb2.conn.rollback()
continue
# Commit the changes
self.windb2.conn.commit()
# Calaculate the insert rate
elapsedTime = (datetime.now() - startTime).seconds
try:
print('Inserted {}, {}-m height x,y wind points at {} I/s'.
format(counter, height_array[z],
counter / elapsedTime))
except ZeroDivisionError:
print('Inserted {}, {}-m height x,y wind points'.format(
counter, height_array[z]))
except UnboundLocalError:
print('Inserted {}, {}-m height x,y wind points'.format(
counter, height_array[0]))
# Close the tempfile so it is deleted
tempFile.close()
# Increment the time
tCount += 1
return timeValuesToReturn, domain_key
def insertNcFile(windb2_conn, ncFile, domainKey=None, tableName="current", replaceData=False, sqlWhere="true"):
# Connect to the WinDB
inserter = insert.Insert(windb2_conn)
# Open the tide netCDF file
print('netCDF file type passed to suntans.insertNcFile=', type(ncFile))
if type(ncFile) != nc.netcdf_file:
ncFile = nc.netcdf_file(ncFile, 'r')
# Get the grid dimensions and coordinates
nLong = ncFile.dimensions['west_east']
nLat = ncFile.dimensions['south_north']
nTime = ncFile.variables['Times'].shape[0] # 'Time' dim is UNLIMITED
lonArr = ncFile.variables['utm_easting']
latArr = ncFile.variables['utm_northing']
timeArr = ncFile.variables['Times']
u = ncFile.variables['u_top1m']
v = ncFile.variables['v_top1m']
# Create a new domain if necessary
if domainKey == None:
domainKey = str(inserter.create_new_domain("SF Bay currents ", "SUNTANS", 200, 'm'))
inserter.insert_horiz_geom(domainKey, lonArr, latArr, 26910) # EPSG:26910: NAD83 / UTM zone 10N
inserter.create_new_table(domainKey, tableName, ("speed", "direction"), ("real", "smallint"),
("speed_positive","direction_degree"), ("speed >= 0","direction >= 0 AND direction < 360"))
else:
# Make sure it's a string so that we don't have concatenation problems later
domainKey = str(domainKey)
# Get the geomkeys associated with the WRF coordinates
horizGeomKey = inserter.calculateHorizWindGeomKeys(domainKey, nLong, nLat)
# Create a counter to execute every so often
counter = 0
startTime = datetime.now()
# Pass the timearr through the timearr filter, even if no filter is set to return Postres
# compliant timestamps. See Python datetime.datetime for datetime format details
timesToInsert = windb2_conn.filterTimes(timeArr, '%Y-%m-%d_%H:%M:%S', sqlWhere=sqlWhere)
# Info
print('Reduced the number of times to insert by ',
round((1 - float(len(timesToInsert)) / timeArr.shape[0]) * 100, 1), '%')
# Iterate through the times that we want to insert
ttiCount = 0
tncfCount = 0
timeValuesToReturn = []
while ttiCount < len(timesToInsert) and tncfCount < nTime:
# Only insert if this is a time we want to insert
tncf = datetime.strptime(timeArr[tncfCount].tostring().decode('UTF-8'), '%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc)
tti = timesToInsert[ttiCount]
if tti != tncf:
tncfCount += 1
continue
# Open a temporary file to COPY from
temp_file = tempfile.NamedTemporaryFile(mode='w+b')
# Create the time in GeoServer/GeoWebCache format
timeValuesToReturn.append(tncf.strftime('%Y-%m-%dT%H:%M:%S.000Z'))
# Info
print('Processing time: ', timeValuesToReturn[-1])
# Iterate through the x,y, and timearr and insert the tidal current data
for x in range(horizGeomKey.shape[0]):
for y in range(horizGeomKey.shape[1]):
# Make sure that this is actually a x,y point we want to insert
# In order to create a mask of selective insert points, all
# a horizGeomKey of zero means we don't want to insert this one
if horizGeomKey[x,y]==0:
continue;
# Write the data string to the temp file
if not numpy.isnan(u[tncfCount,y,x]):
# Calculate speed
speed = math.sqrt(math.pow(u[tncfCount,y,x],2) + math.pow(v[tncfCount,y,x],2))
# Calculate direction (using the 'flow' convention for tides)
dir = int(util.calc_dir_deg(u[tncfCount,y,x], v[tncfCount,y,x]))
# Add this row to be inserted into the database
str_to_write = '{},{},{},{},{},{}\n'.format(domainKey, horizGeomKey[x,y],
tncf.strftime('%Y-%m-%d %H:%M:%S %Z'), speed, dir, 0)
temp_file.write(bytes(str_to_write, 'utf-8'))
counter += 1
# Insert the data at height 0 for tidal current
temp_file.flush()
insertColumns = ('domainkey', 'geomkey', 't', 'speed', 'direction', 'height')
try:
windb2_conn.curs.copy_from(open(temp_file.name, 'rb'), tableName + '_' + domainKey, sep=',', columns=insertColumns)
except psycopg2.IntegrityError as e:
# Delete the duplicate data
errorTest = 'duplicate key value violates unique constraint "' + tableName + "_" + domainKey + '_domainkey_geomkey_t_height_key"'
if re.search(errorTest, str(e)):
# Delete the data and retry the insert if asked to replace data in the function call
if replaceData:
# Rollback to the last commit (necessary to reset the database connection)
windb2_conn.conn.rollback()
# Delete that timearr (assumes UTC timearr zone)
sql = 'DELETE FROM ' + tableName + '_' + domainKey + ' WHERE t = timestamp with time zone\'' + \
tti.strftime('%Y-%m-%d %H:%M:%S %Z') + '\' ' + \
'AND height=0'
print("Deleting conflicting times: " + sql)
windb2_conn.curs.execute(sql)
windb2_conn.conn.commit()
# Reinsert that timearr
windb2_conn.curs.copy_from(open(temp_file.name, 'r'), tableName + '_' + domainKey, sep=',', columns=insertColumns)
# Otherwise, just notify that the insert failed because of duplicate data
else:
logging.error("ERROR ON INSERT: ", e.message)
logging.error("Use 'replaceData=True' if you want the data to be reinserted.")
raise
# Commit the changes
windb2_conn.conn.commit()
# Calaculate the insert rate
elapsedTime = (datetime.now() - startTime).seconds
if elapsedTime > 0:
insertRate = counter / elapsedTime
print("Inserted ", counter, " x,y wind points at ", insertRate, " I/s")
# Close the tempfile so it is deleted
temp_file.close()
# Increment the time
ttiCount += 1
return timeValuesToReturn