def delete_duplicates():
setup_env()
db_server = os.getenv("DB_SERVER")
db_port = os.getenv("DB_PORT")
db_username = os.getenv("DB_USERNAME")
db_password = os.getenv("DB_PASSWORD")
db_database = os.getenv("DB_DATABASE")
config = f"host='{db_server}' port={db_port} user='******' password='******' dbname='{db_database}'"
try:
conn = psycopg2.connect(config)
print("Connection to database established.")
except:
print("Could not connect to the database!")
with conn.cursor() as cur:
print("Deleting duplicates...")
cur.execute(
"DELETE FROM radolan_data AS a USING radolan_data AS b WHERE a.id < b.id AND a.grid_id = b.grid_id AND a.time = b.time")
conn.commit()
print("Deleted duplicates.")
conn.close()
print("Connection closed.")
def crop_data():
setup_env()
temp_path = os.getenv("TEMP_PATH")
boundaries_path = os.getenv("BOUNDARIES_PATH")
bounds = os.getenv("BOUNDS")
print("Starting cropping...")
filelist = create_filelist()
options = gdal.WarpOptions(resampleAlg=gdal.GRA_NearestNeighbour,
format="GTiff",
cutlineDSName=boundaries_path + "/" + bounds + "/" + bounds + ".shp",
dstSRS="+proj=stere +lon_0=10.0 +lat_0=90.0 +lat_ts=60.0 +a=6370040 +b=6370040 +units=m",
srcSRS="+proj=stere +lon_0=10.0 +lat_0=90.0 +lat_ts=60.0 +a=6370040 +b=6370040 +units=m",
cropToCutline=True)
for file in tqdm(filelist, unit=" files"):
file_split = file.split("/")
date_time_obj = datetime.strptime(
file_split[len(file_split)-1], 'RW_%Y%m%d-%H%M.asc')
_ = gdal.Warp(
temp_path + "/cropped/{}.tif".format(date_time_obj.strftime("%Y%m%d-%H%M")), file, options=options)
_ = None
print("Cropping complete.")
def get_buffer_shp():
setup_env()
boundaries_path = os.getenv("BOUNDARIES_PATH")
boundaries_shp_file = os.getenv("BOUNDARIES_SHP_FILE")
bounds = os.getenv("BOUNDS")
temp_path = os.getenv("TEMP_PATH")
boundaries = geopandas.read_file(
boundaries_path + "/" + bounds + "/" + bounds
)
boundaries = boundaries.to_crs("epsg:3857")
boundaries = geopandas.GeoDataFrame(
geopandas.GeoSeries(
unary_union(
boundaries['geometry']
)
)
)
boundaries = boundaries.rename(
columns={0: 'geometry'}).set_geometry('geometry')
boundaries = boundaries.buffer(2000)
boundaries = boundaries.simplify(1000)
# store for later use
boundaries = geopandas.GeoDataFrame(boundaries)
boundaries = boundaries.rename(
columns={0: 'geometry'}).set_geometry('geometry')
boundaries.crs = "epsg:3857"
boundaries.to_file(temp_path + "/" + bounds + ".shp")
def transfer_temp_data():
setup_env()
db_server = os.getenv("DB_SERVER")
db_port = os.getenv("DB_PORT")
db_username = os.getenv("DB_USERNAME")
db_password = os.getenv("DB_PASSWORD")
db_database = os.getenv("DB_DATABASE")
config = f"host='{db_server}' port={db_port} user='******' password='******' dbname='{db_database}'"
try:
conn = psycopg2.connect(config)
print("Connection to database established.")
except:
print("Could not connect to the database!")
with conn.cursor() as cur:
print("Starting to transfer temp data...")
cur.execute('INSERT INTO radolan_data ("createdAt", "updatedAt", grid_id, value, time) SELECT CURRENT_TIMESTAMP(0), CURRENT_TIMESTAMP(0), gridgeoms.id, radolan_temps.value, radolan_temps.time FROM gridgeoms JOIN radolan_temps ON ST_WithIn(gridgeoms.centroid, radolan_temps.geom);')
cur.execute('DELETE FROM radolan_temps;')
conn.commit()
conn.close()
print("Connection closed.")
def upload_temp_data(values):
setup_env()
db_server = os.getenv("DB_SERVER")
db_port = os.getenv("DB_PORT")
db_username = os.getenv("DB_USERNAME")
db_password = os.getenv("DB_PASSWORD")
db_database = os.getenv("DB_DATABASE")
config = f"host='{db_server}' port={db_port} user='******' password='******' dbname='{db_database}'"
try:
conn = psycopg2.connect(config)
print("Connection to database established.")
except:
print("Could not connect to the database!")
try:
print(f"Starting to upload temp data '{values[0][2]}'...")
with conn.cursor() as cur:
cur.execute('DELETE FROM radolan_temps;')
psycopg2.extras.execute_batch(
cur,
'INSERT INTO radolan_temps (geom, value, time, "createdAt", "updatedAt") VALUES (ST_GeomFromText(%s, 4326), %s, %s, CURRENT_TIMESTAMP(0), CURRENT_TIMESTAMP(0));',
values
)
conn.commit()
except:
print("No rain! Continuing with the next dataset.")
conn.close()
print("Connection closed.")
def create_dataframe():
setup_env()
temp_path = os.getenv("TEMP_PATH")
filelist = create_filelist()
for file in tqdm(filelist, unit=".shp-file"):
values = []
file_split = file.split("/")
date_time_obj = datetime.strptime(
file_split[len(file_split)-1], 'RW_%Y%m%d-%H%M.asc')
filename = temp_path + "/vectorized/{}".format(
date_time_obj.strftime("%Y%m%d-%H%M"))
df = geopandas.read_file(filename + ".shp")
df = df.to_crs("epsg:4326")
if df['geometry'].count() > 0:
notNullValues = df[(df['rain'] > 0) & (df['rain'].notnull())]
if len(notNullValues) > 0:
number_of_rows = len(notNullValues.index)
for _, row in tqdm(notNullValues.iterrows(), leave=False, total=number_of_rows, unit=" rows"):
values.append(
[dumps(row.geometry, rounding_precision=5), row.rain, date_time_obj])
upload_temp_data(values)
transfer_temp_data()
delete_duplicates()
df = None
values = None
def upload_data(values):
setup_env()
db_server = os.getenv("DB_SERVER")
db_port = os.getenv("DB_PORT")
db_username = os.getenv("DB_USERNAME")
db_password = os.getenv("DB_PASSWORD")
db_database = os.getenv("DB_DATABASE")
config = f"host='{db_server}' port={db_port} user='******' password='******' dbname='{db_database}'"
try:
conn = psycopg2.connect(config)
print("Connection to database established.")
except:
print("Could not establish database connection!")
print("Uploading data...")
with conn.cursor() as cur:
psycopg2.extras.execute_batch(
cur,
'INSERT INTO gridgeoms (geom, centroid, "createdAt", "updatedAt") VALUES (ST_GeomFromText(%s, 4326), ST_Centroid(ST_GeomFromText(%s, 4326)), CURRENT_TIMESTAMP(0), CURRENT_TIMESTAMP(0));',
values
)
conn.commit()
print("Data uploaded.")
conn.close()
print("Connection closed.")
def cleanup():
setup_env()
temp_path = os.getenv("TEMP_PATH")
shutil.rmtree(temp_path, ignore_errors=True)
print("Cleanup complete.")
def create_filelist():
setup_env()
temp_path = os.getenv("TEMP_PATH")
filelist = []
path = temp_path + "/unpacked"
for (_, dirnames, _) in os.walk(path):
for dirname in dirnames:
dpath = path + "/" + dirname
for (_, _, ffilenames) in os.walk(dpath):
for ffilename in ffilenames:
filelist.append(dpath + "/" + ffilename)
return filelist
def vectorize_data():
setup_env()
temp_path = os.getenv("TEMP_PATH")
filelist = create_filelist()
print("Starting vectorization...")
for file in tqdm(filelist, unit=" file"):
file_split = file.split("/")
date_time_obj = datetime.strptime(
file_split[len(file_split)-1], 'RW_%Y%m%d-%H%M.asc')
filename_input = temp_path + "/cropped/{}".format(
date_time_obj.strftime("%Y%m%d-%H%M"))
filename_output = temp_path + "/vectorized/{}".format(
date_time_obj.strftime("%Y%m%d-%H%M"))
source = gdal.Open(filename_input + ".tif")
band = source.GetRasterBand(1)
_ = band.ReadAsArray()
driver = ogr.GetDriverByName("ESRI Shapefile")
if os.path.exists(filename_output + ".shp"):
driver.DeleteDataSource(filename_output + ".shp")
target = driver.CreateDataSource(filename_output + ".shp")
srs = osr.SpatialReference()
srs.ImportFromProj4(
"+proj=stere +lon_0=10.0 +lat_0=90.0 +lat_ts=60.0 +a=6370040 +b=6370040 +units=m")
targetLayer = target.CreateLayer("radolan", srs=srs)
targetField = ogr.FieldDefn("rain", ogr.OFTInteger)
targetLayer.CreateField(targetField)
gdal.Polygonize(band, None, targetLayer, 0, [], callback=None)
target.Destroy()
source = None
_ = None
print("Vectorization complete.")
def create_values():
setup_env()
radolan_path = os.getenv("RADOLAN_PATH")
radolan_grid_path = radolan_path + '/radolan_grid'
values = []
print("Creating dataframe...")
radolan_grid = gpd.read_file(radolan_grid_path).to_crs("epsg:4326")
print("Dataframe created.")
print("Creating values list...")
for _, row in tqdm(radolan_grid.iterrows(), leave=False, total=len(radolan_grid), unit=" rows"):
geom = dumps(row.geometry, rounding_precision=5)
values.append([
geom, geom
])
print("Values list created.")
return values
from flask import Flask, render_template, url_for, request
from flask import redirect, flash
from flask import jsonify, json
# custom imports
from city import City
from city_types import Location
import external
import valid_cities
import city_desc
import email_support
import setup
# variables
app = Flask(__name__)
setup.setup_env(app)
# show index.html
@app.route('/')
def show_index():
return render_template('index.html')
# handle contact form
@app.route('/contact', methods=['GET', 'POST'])
def contact():
if request.method == 'GET':
return render_template('contact.html')
else:
name = request.form['name']
sender = (name, request.form['email-address'])