def modify_df(self,df):
"""Adds a geometry column extracted from the other columns"""
self.dtypes={}
#change all panda object types to ocnvert to a Sqlalcehmy String
for ky,typ in df.dtypes.items():
if type(object) == typ:
self.dtypes[ky]=String
if re.search('A-GLACIER.csv',self.pdfile):
#extract longitude and latitude
self.dtypes["geom"]=geoPointtype
df.LONGITUDE=df.LONGITUDE.fillna(0)
df.LATITUDE=df.LATITUDE.fillna(0)
df['geom']=[WKTElement(wktdump(Point(lon,lat,0)),srid=4326,extended=True) for lon,lat in zip(df.LONGITUDE,df.LATITUDE)]
if re.search('EEE-MASS-BALANCE-POINT.csv',self.pdfile):
#extract longitude and latitude
df.POINT_LON=df.POINT_LON.fillna(0)
df.POINT_LAT=df.POINT_LAT.fillna(0)
df.POINT_ELEVATION=df.POINT_ELEVATION.fillna(0)
df['geom']=[WKTElement(wktdump(Point(lon,lat,h)),srid=4326,extended=True) for lon,lat,h in zip(df.POINT_LON,df.POINT_LAT,df.POINT_ELEVATION)]
self.dtypes["geom"]=geoPointtype
return df
def modify_df(self,df):
"""Adds a geometry column TODO: converts day,month,year column to proper dates"""
#make a geometry column with points
df.altitude=df.altitude.replace(-999,0)
df['geom']=[WKTElement(wktdump(Point(lon,lat,h)),srid=4326,extended=True) for lon,lat,h in zip(df.long,df.lat,df.altitude)]
# gdf = gpd.GeoDataFrame(
# df, geometry=gpd.points_from_xy(df.Longitude, df.Latitude))
return df
def extractMetaPies(datafile):
"""Co routine which spits outs meta info per PIES-site"""
dat=scipy.io.loadmat(datafile)
names=[3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17]
for i in range(dat["OLENGTH"].shape[1]):
depth=float(dat["DEPTH"][0][i])
pnt=Point(dat["degE"][i][0],dat["degN"][i][0])
varlookup={"OPRES":(i,(0,-1,1))}
meta={"name":str(names[i]),"depth":depth,"tstart":decyear2dt(dat["OTMIN"][0][i]),
"tend":decyear2dt(dat["OTMAX"][0][i]),"vars":varlookup,
"uri":datafile, "geom":WKTElement(wktdump(pnt))}
yield meta
def GRDCmetaExtractor(uri):
encoding = 'iso-8859-1'
meta = {}
headerregex = re.compile(b"^#")
dataregex = re.compile(b"^[0-9]")
header = {}
epoch = []
slurplogger().info("Extracting meta info from: %s" % (uri.url))
with gzip.open(uri.url, 'r') as gzid:
for ln in gzid:
if headerregex.search(ln):
#parse header
lnspl = ln[1:].split(b":", 1)
if len(lnspl) == 2:
ky = lnspl[0].lstrip(b" ").rstrip(b" ").decode(encoding)
val = lnspl[1].lstrip(b" ").rstrip(b"\r\n ").decode(
encoding)
if not val:
#get the nextline as a value
val = gzid.readline()[1:].lstrip(b" ").rstrip(
b"\r\n ").decode(encoding)
header[ky] = val
continue
if dataregex.search(ln):
#parse data line
lnspl = ln.decode(encoding).split(";")
datestr = lnspl[0] + lnspl[1].replace("--:--", ":00:00")
epoch.append(datetime.strptime(datestr, '%Y-%m-%d:%H:%S'))
nextdownstream = header["Next downstream station"]
if "-" in nextdownstream:
nextdownstream = None
else:
nextdownstream = int(nextdownstream)
area = float(header["Catchment area (km²)"])
if area == -999.0:
area = None
alt = float(header["Altitude (m ASL)"])
if alt == -999.0:
alt = None
location = Point(float(header["Longitude (DD)"]),
float(header["Latitude (DD)"]), 0)
else:
location = Point(float(header["Longitude (DD)"]),
float(header["Latitude (DD)"]), alt)
# import pdb;pdb.set_trace()
try:
meta = {
"grdc_no": int(header["GRDC-No."]),
"tstart": epoch[0],
"tend": epoch[-1],
"lastupdate": datetime.strptime(header["Last update"], "%Y-%m-%d"),
"area": area,
"altitude": alt,
"river": header["River"],
"statname": header["Station"],
"country": header["Country"],
"nextstation": nextdownstream,
"uri": uri.url,
"geom": WKTElement(wktdump(location), srid=4326, extended=True)
}
# "geom":WKBElement(wkbdump(location),srid=4326,extended=True)}
except KeyError as e:
import pdb
pdb.set_trace()
raise (e)
return meta
def geom2wkt(cls, geom):
return wktdump(geom)