def testCalcDirDegrees(self): # First quadrant self.assertEqual(util.calc_dir_deg(3, 3), 45) # Second quadrant self.assertEqual(util.calc_dir_deg(3, -3), 135) # Third quadrant self.assertEqual(util.calc_dir_deg(-3, -3), 225) # Fourth quadrant self.assertEqual(util.calc_dir_deg(-3, 3), 315)
def uv_column_interp(u_mass_column, v_mass_column, heights_above_ground, heights_to_interp, z_o=None): """Calculate the wind speeds at a given height (in meters) using the two adjacent pressure levels u and v wind components. Uses a log-law approximation from the ground to 100 m for points that are lower than the lowest eta level in the model. Direction interpolation is not performed for levels where the log-law approximation is used. :param u_mass_column: WRF U wind speed already interpolated to the mass coordinate system (native is flux coordinate) :param v_mass_column: WRF V wind speed already interpolated to the mass coordinate system (native is flux coordinate) :param heights_above_ground: :param heights_to_interp: :returns 2, 1D arrays u, v of the height interpolated wind speed and direction in the column. """ # Make sure array lengths match assert(u_mass_column.shape == v_mass_column.shape) assert(u_mass_column.shape == heights_above_ground.shape) # Make sure we've only been given a column assert(u_mass_column.ndim == 1) assert(v_mass_column.ndim == 1) assert(heights_above_ground.ndim == 1) # Convert heights_to_interp to an numpy array so we can mask it heights_to_interp = numpy.array(heights_to_interp) # Find the heights that need log-law interpolation i.e. below 100 m and lower than the lowest eta-half level mask_heights_to_interp_log_law = heights_to_interp < numpy.min(heights_above_ground) # Linear interp where we have model data if numpy.size(heights_to_interp[~mask_heights_to_interp_log_law]) > 0: u_mass_coord_interp_linear = numpy.interp(heights_to_interp[~mask_heights_to_interp_log_law], heights_above_ground, u_mass_column) v_mass_coord_interp_linear = numpy.interp(heights_to_interp[~mask_heights_to_interp_log_law], heights_above_ground, v_mass_column) else: logger.info('No eta-level heights were in the range of LINEAR interpolation.') u_mass_coord_interp_linear = [] v_mass_coord_interp_linear = [] # Log-law interp for speed using heights 100 m and below, if WRF ZNT was NOT provided speed = util.speed(u_mass_column, v_mass_column) if numpy.size(heights_to_interp[mask_heights_to_interp_log_law]) > 0 and z_o is None: # Make sure there is more than one height to linearly interpolate with below 100 m if heights_above_ground[1] < 100: speed_interp_log_law = log_law_interp(speed, heights_above_ground, heights_to_interp[mask_heights_to_interp_log_law], 100) # Otherwise, issue a warning and include the second height above ground # This situation can sometimes occur at the edges of nested domains when the PSFC value # is lower than the lowest level of (P + PB). Speculate this is a bug in WRF where the PSFC is being taken # from the lower resolution domain rather than being calculated with the new topo height in the higher # resolution domain. else: speed_interp_log_law = log_law_interp(speed, heights_above_ground, heights_to_interp[mask_heights_to_interp_log_law], heights_above_ground[1]) logger.info('Had to use height above 100 m for log-linear log-law interp') logger.info('Heights used for log-linear log-law interp: {} m and {} m' .format(heights_above_ground[0], heights_above_ground[1])) # Use the log-law with the WRF ZNT (surface roughness) to diagnose U,V elif numpy.size(heights_to_interp[mask_heights_to_interp_log_law]) > 0 and z_o is not None: speed_interp_log_law = speed[0] * \ numpy.log(heights_to_interp[mask_heights_to_interp_log_law] / z_o) / \ numpy.log(numpy.min(heights_above_ground) / z_o) else: logger.info('No eta-level heights were in the range of LOG-LAW interpolation.') speed_interp_log_law = [] # For log-law interpolation, use the same wind direction as the bottom level where we have WRF data for the # direction. Assumes these heights are sorted from bottom to top, which is the WRF default. u_bottom = u_mass_column[0] v_bottom = v_mass_column[0] dir_interp_log_law = util.calc_dir_deg(u_bottom, v_bottom) u_mass_coord_interp_log_law = util.u_flow(speed_interp_log_law, dir_interp_log_law) v_mass_coord_interp_log_law = util.v_flow(speed_interp_log_law, dir_interp_log_law) return numpy.hstack((u_mass_coord_interp_log_law, u_mass_coord_interp_linear)), \ numpy.hstack((v_mass_coord_interp_log_law, v_mass_coord_interp_linear))
def insertNcFile(windb2_conn, ncFile, domainKey=None, tableName="current", replaceData=False, sqlWhere="true"): # Connect to the WinDB inserter = insert.Insert(windb2_conn) # Open the tide netCDF file print('netCDF file type passed to suntans.insertNcFile=', type(ncFile)) if type(ncFile) != nc.netcdf_file: ncFile = nc.netcdf_file(ncFile, 'r') # Get the grid dimensions and coordinates nLong = ncFile.dimensions['west_east'] nLat = ncFile.dimensions['south_north'] nTime = ncFile.variables['Times'].shape[0] # 'Time' dim is UNLIMITED lonArr = ncFile.variables['utm_easting'] latArr = ncFile.variables['utm_northing'] timeArr = ncFile.variables['Times'] u = ncFile.variables['u_top1m'] v = ncFile.variables['v_top1m'] # Create a new domain if necessary if domainKey == None: domainKey = str(inserter.create_new_domain("SF Bay currents ", "SUNTANS", 200, 'm')) inserter.insert_horiz_geom(domainKey, lonArr, latArr, 26910) # EPSG:26910: NAD83 / UTM zone 10N inserter.create_new_table(domainKey, tableName, ("speed", "direction"), ("real", "smallint"), ("speed_positive","direction_degree"), ("speed >= 0","direction >= 0 AND direction < 360")) else: # Make sure it's a string so that we don't have concatenation problems later domainKey = str(domainKey) # Get the geomkeys associated with the WRF coordinates horizGeomKey = inserter.calculateHorizWindGeomKeys(domainKey, nLong, nLat) # Create a counter to execute every so often counter = 0 startTime = datetime.now() # Pass the timearr through the timearr filter, even if no filter is set to return Postres # compliant timestamps. See Python datetime.datetime for datetime format details timesToInsert = windb2_conn.filterTimes(timeArr, '%Y-%m-%d_%H:%M:%S', sqlWhere=sqlWhere) # Info print('Reduced the number of times to insert by ', round((1 - float(len(timesToInsert)) / timeArr.shape[0]) * 100, 1), '%') # Iterate through the times that we want to insert ttiCount = 0 tncfCount = 0 timeValuesToReturn = [] while ttiCount < len(timesToInsert) and tncfCount < nTime: # Only insert if this is a time we want to insert tncf = datetime.strptime(timeArr[tncfCount].tostring().decode('UTF-8'), '%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc) tti = timesToInsert[ttiCount] if tti != tncf: tncfCount += 1 continue # Open a temporary file to COPY from temp_file = tempfile.NamedTemporaryFile(mode='w+b') # Create the time in GeoServer/GeoWebCache format timeValuesToReturn.append(tncf.strftime('%Y-%m-%dT%H:%M:%S.000Z')) # Info print('Processing time: ', timeValuesToReturn[-1]) # Iterate through the x,y, and timearr and insert the tidal current data for x in range(horizGeomKey.shape[0]): for y in range(horizGeomKey.shape[1]): # Make sure that this is actually a x,y point we want to insert # In order to create a mask of selective insert points, all # a horizGeomKey of zero means we don't want to insert this one if horizGeomKey[x,y]==0: continue; # Write the data string to the temp file if not numpy.isnan(u[tncfCount,y,x]): # Calculate speed speed = math.sqrt(math.pow(u[tncfCount,y,x],2) + math.pow(v[tncfCount,y,x],2)) # Calculate direction (using the 'flow' convention for tides) dir = int(util.calc_dir_deg(u[tncfCount,y,x], v[tncfCount,y,x])) # Add this row to be inserted into the database str_to_write = '{},{},{},{},{},{}\n'.format(domainKey, horizGeomKey[x,y], tncf.strftime('%Y-%m-%d %H:%M:%S %Z'), speed, dir, 0) temp_file.write(bytes(str_to_write)) counter += 1 # Insert the data at height 0 for tidal current temp_file.flush() insertColumns = ('domainkey', 'geomkey', 't', 'speed', 'direction', 'height') try: windb2_conn.curs.copy_from(open(temp_file.name, 'rb'), tableName + '_' + domainKey, sep=',', columns=insertColumns) except psycopg2.IntegrityError as e: # Delete the duplicate data errorTest = 'duplicate key value violates unique constraint "' + tableName + "_" + domainKey + '_domainkey_geomkey_t_height_key"' if re.search(errorTest, str(e)): # Delete the data and retry the insert if asked to replace data in the function call if replaceData: # Rollback to the last commit (necessary to reset the database connection) windb2_conn.conn.rollback() # Delete that timearr (assumes UTC timearr zone) sql = 'DELETE FROM ' + tableName + '_' + domainKey + ' WHERE t = timestamp with time zone\'' + \ tti.strftime('%Y-%m-%d %H:%M:%S %Z') + '\' ' + \ 'AND height=0' print("Deleting conflicting times: " + sql) windb2_conn.curs.execute(sql) windb2_conn.conn.commit() # Reinsert that timearr windb2_conn.curs.copy_from(open(temp_file.name, 'r'), tableName + '_' + domainKey, sep=',', columns=insertColumns) # Otherwise, just notify that the insert failed because of duplicate data else: logging.error("ERROR ON INSERT: ", e.message) logging.error("Use 'replaceData=True' if you want the data to be reinserted.") raise # Commit the changes windb2_conn.conn.commit() # Calaculate the insert rate elapsedTime = (datetime.now() - startTime).seconds if elapsedTime > 0: insertRate = counter / elapsedTime print("Inserted ", counter, " x,y wind points at ", insertRate, " I/s") # Close the tempfile so it is deleted temp_file.close() # Increment the time ttiCount += 1 return timeValuesToReturn
def insert_variable(self, ncfile, var_name, domain_key=None, replace_data=False, sql_where="true", file_type='windb2', mask=None, zero_seconds=False): """Inserts a netCDF file with WinDB2 or WRF output into a WinDB2 database. * * windb2Conn - Connection to a WinDB2 database. * ncfile - Either an open file or a string name of a file to open. * var_name - Name of WinDB2 supported variable or a WRF 3D variable (currently WIND, THETA, RHO). * domain_key - Existing domain key in the database. If left blank, a new domain will be created. * replace_data - Deletes data for the same time in the database if True. Useful for freshening data. * file_type - Type of netCDF file to insert: {'windb2' (default), or 'wrf'} * mask - String name of a mask in the WinDB2 database. Only relevant when creating a new domain (the mask is * applied automatically thereafter). * * returns timesInsertedList, domain_key - A list of times inserted in ISO time format, and the domain_key where the data was inserted. :param file_type: """ # Make sure this the file_type of file is support if file_type != 'windb2' and file_type != 'wrf': raise TypeError('Unsupported file file_type: {}'.format(file_type)) # Open the WinDB netCDF file logger.debug( 'netCDF file file_type passed to wrf.insertNcFile={}'.format( type(ncfile))) if type(ncfile) != Dataset: ncfile = Dataset(ncfile, 'r') # Get the grid dimensions and coordinates if file_type == 'windb2': nlong = len(ncfile.dimensions['x']) nlat = len(ncfile.dimensions['y']) x_coord_array = ncfile.groups['WRF']['XLONG'][:] y_coord_array = ncfile.groups['WRF']['XLAT'][:] if self.config['vars'][var_name]['dims'] == 3: height_array = ncfile.variables['height'][:] elif self.config['vars'][var_name]['dims'] == 2: height_array = [self.config['vars'][var_name]['insert'][0]] else: raise Exception('Number of dimensions must either be 2 or 3') init_t = datetime.strptime( ncfile.groups['WRF'].SIMULATION_START_DATE, '%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc) elif file_type == 'wrf': nlong = len(ncfile.dimensions['west_east']) nlat = len(ncfile.dimensions['south_north']) x_coord_array = ncfile['XLONG'][:] y_coord_array = ncfile['XLAT'][:] height_array = [self.config[var_name]['insert'][0]] init_t = datetime.strptime( ncfile.SIMULATION_START_DATE, '%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc) # Read in the vars to insert wrf_copied_var = False if file_type == 'windb2' and var_name.lower() == 'WIND'.lower(): u = ncfile.variables['eastward_wind'][:] v = ncfile.variables['northward_wind'][:] elif file_type == 'windb2' and var_name.lower() == 'DPT'.lower(): ncVariable = ncfile.variables['dew_point_temperature'][:] # Otherwise try find the WinDB2 interp or WRF var else: try: ncVariable = ncfile.variables[var_name][:] except KeyError as e: wrf_copied_var = True ncVariable = ncfile.groups['WRF'][var_name][:] # Create a new and/or domain if necessary if domain_key is None: if file_type == 'windb2': domain_key = str( self.create_new_domain(ncfile.groups['WRF'].TITLE, "WRF", ncfile.groups['WRF'].DX, 'm', mask)) elif file_type == 'wrf': domain_key = str( self.create_new_domain(ncfile.TITLE, "WRF", ncfile.DX, 'm', mask)) self.insert_horiz_geom(domain_key, x_coord_array, y_coord_array, create_wrf_srid(self.windb2, ncfile)) # Mask the domain if necessary if mask is not None: self.mask_domain(domain_key, mask) # Create a new table if necessary and add an initialization time column if file_type == 'windb2' and var_name.lower() == 'wind'.lower(): if not self.windb2.table_exists('wind' + '_' + domain_key): self.create_new_table(domain_key, var_name, ('speed', 'direction'), ('real', 'smallint')) self._create_initialization_time_column(var_name, domain_key) else: if not self.windb2.table_exists('{}_{}'.format( var_name.lower(), domain_key)): self.create_new_table(domain_key, var_name, ('value', ), ('real', )) self._create_initialization_time_column( var_name.lower(), domain_key) # Make sure it's a string so that we don't have concatenation problems later domain_key = str(domain_key) # Get the geomkeys associated with the WRF coordinates horizGeomKey = self.calculateHorizWindGeomKeys(domain_key, nlong, nlat) # Create a counter to execute every so often counter = 0 startTime = datetime.now() # Get the time array to iterate through if file_type == 'windb2': time_char_array = chartostring(ncfile.variables['Time'][:]) elif file_type == 'wrf': time_char_array = chartostring(ncfile.variables['Times'][:]) # Create a statement to use tCount = 0 timeValuesToReturn = [] for t in time_char_array: # Create a datetime from the WRF string t = datetime.strptime(t, '%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc) # Zero the seconds if asked to if zero_seconds: t = t.replace(second=0) # Create the time in GeoServer/GeoWebCache format timeValuesToReturn.append(t.strftime('%Y-%m-%dT%H:%M:%S.000Z')) # Info print('Processing time for {}: {}'.format(var_name, timeValuesToReturn[-1])) # Iterate through the x,y, and timearr and insert the WRF variable for h in height_array: # We actually need the index of the height, not the actual height itself height = None if file_type == 'windb2' and wrf_copied_var is False: try: z = numpy.argwhere( ncfile.variables['height'][:] == h)[0, 0] height = height_array[z] except IndexError: logger.error( 'Height {} to insert does not exist in WinDB2 file' .format(h)) sys.exit(-1) elif wrf_copied_var is True: height = height_array[0] else: height = 0 counter = 0 # Open a temporary file to COPY from tempFile = tempfile.NamedTemporaryFile(mode='w') for x in range(horizGeomKey.shape[0]): for y in range(horizGeomKey.shape[1]): # Make sure that this is actually a x,y point we want to insert # In order to create a mask of selective insert points, all # a horizGeomKey of zero means we don't want to insert this one if horizGeomKey[x, y] == 0: continue # Write the data string to the temp file if file_type == 'windb2' and var_name.lower( ) == 'wind'.lower(): if not (numpy.isnan(u[tCount, z, y, x]) or numpy.isnan(v[tCount, z, y, x])): # Add this row to be inserted into the database # Note that we negate U and V so they exist in WinDB2 as the vernacular "coming from" wind direction print('{}, {}, {}, {}, {}, {}, {}'.format( domain_key, horizGeomKey[x, y], t.strftime('%Y-%m-%d %H:%M:%S %Z'), util.speed(u[tCount, z, y, x], v[tCount, z, y, x]), int( util.calc_dir_deg( -u[tCount, z, y, x], -v[tCount, z, y, x])), height, init_t.strftime('%Y-%m-%d %H:%M:%S %Z')), file=tempFile) counter += 1 elif file_type == 'windb2': # Add this row to be inserted into the database if self.config['vars'][var_name]['dims'] == 2: val = ncVariable[tCount, y, x] elif self.config['vars'][var_name]['dims'] == 3: val = ncVariable[tCount, z, y, x] if not numpy.isnan(val): print('{}, {}, {}, {}, {}, {}'.format( domain_key, horizGeomKey[x, y], t.strftime('%Y-%m-%d %H:%M:%S %Z'), val, height, init_t.strftime('%Y-%m-%d %H:%M:%S %Z')), file=tempFile) counter += 1 elif file_type == 'wrf': if not numpy.isnan(ncVariable[tCount, y, x]): # Add this row to be inserted into the database print('{}, {}, {}, {}, {}'.format( domain_key, horizGeomKey[x, y], t.strftime('%Y-%m-%d %H:%M:%S %Z'), ncVariable[tCount, y, x], init_t.strftime('%Y-%m-%d %H:%M:%S %Z')), file=tempFile) counter += 1 # Insert the data tempFile.flush() if file_type == 'windb2' and var_name.lower() == 'wind'.lower( ): insertColumns = columns = ('domainkey', 'geomkey', 't', 'speed', 'direction', 'height', 'init') else: insertColumns = columns = ('domainkey', 'geomkey', 't', 'value', 'height', 'init') try: self.windb2.curs.copy_from(open(tempFile.name, 'r'), var_name + '_' + domain_key, sep=',', columns=insertColumns) except psycopg2.IntegrityError as e: # Delete the duplicate data errorTest = 'duplicate key value violates unique constraint "' + var_name.lower( ) + "_" + domain_key + '_domainkey_geomkey_t_height_init_key"' if re.search(errorTest, str(e.pgerror)): # Delete the data and retry the insert if asked to replace data in the function call if replace_data: # Rollback to the last commit (necessary to reset the database connection) self.windb2.conn.rollback() # Delete that timearr (assumes UTC timearr zone) sql = 'DELETE FROM ' + var_name + '_' + domain_key + \ ' WHERE t = timestamp with time zone\'' + t.strftime('%Y-%m-%d %H:%M:%S %Z') + '\' ' + \ 'AND height=' + str(h) print("Deleting conflicting times: " + sql) self.windb2.curs.execute(sql) self.windb2.conn.commit() # Reinsert that timearr self.windb2.curs.copy_from( open(tempFile.name, 'r'), var_name + '_' + domain_key, sep=',', columns=insertColumns) # Commit again or the reinserts won't stick self.windb2.conn.commit() continue # Otherwise, just notify that the insert failed because of duplicate data. We do re-raise this error # because it's assumed that we want to suplement the WinDB with other data-heights if available. else: logging.warning('ERROR ON INSERT: {}'.format( e.pgerror)) logging.warning( 'Use \'replace_data=True\' if you want the data to be reinserted.' ) self.windb2.conn.rollback() continue # Commit the changes self.windb2.conn.commit() # Calaculate the insert rate elapsedTime = (datetime.now() - startTime).seconds try: print('Inserted {}, {}-m height x,y wind points at {} I/s'. format(counter, height_array[z], counter / elapsedTime)) except ZeroDivisionError: print('Inserted {}, {}-m height x,y wind points'.format( counter, height_array[z])) except UnboundLocalError: print('Inserted {}, {}-m height x,y wind points'.format( counter, height_array[0])) # Close the tempfile so it is deleted tempFile.close() # Increment the time tCount += 1 return timeValuesToReturn, domain_key
def insertNcFile(windb2_conn, ncFile, domainKey=None, tableName="current", replaceData=False, sqlWhere="true"): # Connect to the WinDB inserter = insert.Insert(windb2_conn) # Open the tide netCDF file print('netCDF file type passed to suntans.insertNcFile=', type(ncFile)) if type(ncFile) != nc.netcdf_file: ncFile = nc.netcdf_file(ncFile, 'r') # Get the grid dimensions and coordinates nLong = ncFile.dimensions['west_east'] nLat = ncFile.dimensions['south_north'] nTime = ncFile.variables['Times'].shape[0] # 'Time' dim is UNLIMITED lonArr = ncFile.variables['utm_easting'] latArr = ncFile.variables['utm_northing'] timeArr = ncFile.variables['Times'] u = ncFile.variables['u_top1m'] v = ncFile.variables['v_top1m'] # Create a new domain if necessary if domainKey == None: domainKey = str(inserter.create_new_domain("SF Bay currents ", "SUNTANS", 200, 'm')) inserter.insert_horiz_geom(domainKey, lonArr, latArr, 26910) # EPSG:26910: NAD83 / UTM zone 10N inserter.create_new_table(domainKey, tableName, ("speed", "direction"), ("real", "smallint"), ("speed_positive","direction_degree"), ("speed >= 0","direction >= 0 AND direction < 360")) else: # Make sure it's a string so that we don't have concatenation problems later domainKey = str(domainKey) # Get the geomkeys associated with the WRF coordinates horizGeomKey = inserter.calculateHorizWindGeomKeys(domainKey, nLong, nLat) # Create a counter to execute every so often counter = 0 startTime = datetime.now() # Pass the timearr through the timearr filter, even if no filter is set to return Postres # compliant timestamps. See Python datetime.datetime for datetime format details timesToInsert = windb2_conn.filterTimes(timeArr, '%Y-%m-%d_%H:%M:%S', sqlWhere=sqlWhere) # Info print('Reduced the number of times to insert by ', round((1 - float(len(timesToInsert)) / timeArr.shape[0]) * 100, 1), '%') # Iterate through the times that we want to insert ttiCount = 0 tncfCount = 0 timeValuesToReturn = [] while ttiCount < len(timesToInsert) and tncfCount < nTime: # Only insert if this is a time we want to insert tncf = datetime.strptime(timeArr[tncfCount].tostring().decode('UTF-8'), '%Y-%m-%d_%H:%M:%S').replace(tzinfo=pytz.utc) tti = timesToInsert[ttiCount] if tti != tncf: tncfCount += 1 continue # Open a temporary file to COPY from temp_file = tempfile.NamedTemporaryFile(mode='w+b') # Create the time in GeoServer/GeoWebCache format timeValuesToReturn.append(tncf.strftime('%Y-%m-%dT%H:%M:%S.000Z')) # Info print('Processing time: ', timeValuesToReturn[-1]) # Iterate through the x,y, and timearr and insert the tidal current data for x in range(horizGeomKey.shape[0]): for y in range(horizGeomKey.shape[1]): # Make sure that this is actually a x,y point we want to insert # In order to create a mask of selective insert points, all # a horizGeomKey of zero means we don't want to insert this one if horizGeomKey[x,y]==0: continue; # Write the data string to the temp file if not numpy.isnan(u[tncfCount,y,x]): # Calculate speed speed = math.sqrt(math.pow(u[tncfCount,y,x],2) + math.pow(v[tncfCount,y,x],2)) # Calculate direction (using the 'flow' convention for tides) dir = int(util.calc_dir_deg(u[tncfCount,y,x], v[tncfCount,y,x])) # Add this row to be inserted into the database str_to_write = '{},{},{},{},{},{}\n'.format(domainKey, horizGeomKey[x,y], tncf.strftime('%Y-%m-%d %H:%M:%S %Z'), speed, dir, 0) temp_file.write(bytes(str_to_write, 'utf-8')) counter += 1 # Insert the data at height 0 for tidal current temp_file.flush() insertColumns = ('domainkey', 'geomkey', 't', 'speed', 'direction', 'height') try: windb2_conn.curs.copy_from(open(temp_file.name, 'rb'), tableName + '_' + domainKey, sep=',', columns=insertColumns) except psycopg2.IntegrityError as e: # Delete the duplicate data errorTest = 'duplicate key value violates unique constraint "' + tableName + "_" + domainKey + '_domainkey_geomkey_t_height_key"' if re.search(errorTest, str(e)): # Delete the data and retry the insert if asked to replace data in the function call if replaceData: # Rollback to the last commit (necessary to reset the database connection) windb2_conn.conn.rollback() # Delete that timearr (assumes UTC timearr zone) sql = 'DELETE FROM ' + tableName + '_' + domainKey + ' WHERE t = timestamp with time zone\'' + \ tti.strftime('%Y-%m-%d %H:%M:%S %Z') + '\' ' + \ 'AND height=0' print("Deleting conflicting times: " + sql) windb2_conn.curs.execute(sql) windb2_conn.conn.commit() # Reinsert that timearr windb2_conn.curs.copy_from(open(temp_file.name, 'r'), tableName + '_' + domainKey, sep=',', columns=insertColumns) # Otherwise, just notify that the insert failed because of duplicate data else: logging.error("ERROR ON INSERT: ", e.message) logging.error("Use 'replaceData=True' if you want the data to be reinserted.") raise # Commit the changes windb2_conn.conn.commit() # Calaculate the insert rate elapsedTime = (datetime.now() - startTime).seconds if elapsedTime > 0: insertRate = counter / elapsedTime print("Inserted ", counter, " x,y wind points at ", insertRate, " I/s") # Close the tempfile so it is deleted temp_file.close() # Increment the time ttiCount += 1 return timeValuesToReturn