def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
csv_reader = csv.DictReader(f,
delimiter=',',
quoting=csv.QUOTE_ALL)
townships_interpolated = [{
"code":
row['id'],
"name":
row['name'],
"township":
row['township'],
"year":
datetime.strptime(row['year'], "%Y-%m-%d %H:%M:%S").year,
self.interpolated_value_name:
Decimal(row['interpolated_values'])
} for row in csv_reader]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.bulk_insert_mappings(mapper=self.model,
mappings=townships_interpolated,
render_nulls=True,
return_defaults=False)
session.commit()
session.close()
def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
csv_reader = csv.DictReader(
f, delimiter=',',
quoting=csv.QUOTE_NONE) # quote non to skip whitespace
weather_station_data = [
dict(station_id=int(row['station_id']),
date=datetime.strptime(row['date'], "%Y-%m-%d"),
temperature_avg=Decimal(row['temperature_avg']),
temperature_min=Decimal(row['temperature_min']),
temperature_max=Decimal(row['temperature_max']),
sunshine_duration=Decimal(row['sunshine_duration']),
sunshine_radiation=Decimal(row['sunshine_radiation']),
rain_duration=Decimal(row['rain_duration']),
rain_sum=Decimal(row['rain_sum']),
humidity_avg=Decimal(row['humidity_avg']),
humidity_max=Decimal(row['humidity_max']),
humidity_min=Decimal(row['humidity_min']))
for row in csv_reader
]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.bulk_insert_mappings(mapper=WeatherStationDataObject,
mappings=weather_station_data,
render_nulls=True,
return_defaults=False)
session.commit()
session.close()
def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
csv_reader = csv.DictReader(
f, delimiter=',',
quoting=csv.QUOTE_NONE) # quote non to skip whitespace
trees = [
dict(
# species_latin=row['species_latin'],
species_dutch=row['species_dutch'],
geometry=row['geometry'],
origin='amsterdam') for row in csv_reader
]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.bulk_insert_mappings(mapper=TreeObject,
mappings=trees,
render_nulls=True,
return_defaults=False)
session.commit()
session.close()
def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
csv_reader = csv.DictReader(
f, delimiter=',',
quoting=csv.QUOTE_NONE) # quote non to skip whitespace
oak_processionary_moths = [
dict(date=row['date'],
stage=row['stage'],
geometry=row['geometry'],
origin='vlinderstichting',
granularity='moth') for row in csv_reader
]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.bulk_insert_mappings(mapper=OakProcessionaryMothObject,
mappings=oak_processionary_moths,
render_nulls=True,
return_defaults=False)
session.commit()
session.close()
def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
csv_reader = csv.DictReader(f,
delimiter=',',
quoting=csv.QUOTE_ALL)
neighbourhoods = [
dict(code=row['id'],
name=row['name'],
township=row['township'],
geometry=row['geometry'],
area=float(row['area'])) for row in csv_reader
]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.bulk_insert_mappings(mapper=NeighbourhoodObject,
mappings=neighbourhoods,
render_nulls=True,
return_defaults=False)
session.commit()
session.close()
def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
json_file = json.load(f)
townships = [
TownshipObject(name=line['properties']['name'],
code=line['properties']['code'],
geometry=shape(line['geometry']).wkt)
for line in json_file['features']
]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.add_all(townships)
session.commit()
session.close()
def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
csv_reader = csv.DictReader(f,
delimiter=',',
quoting=csv.QUOTE_ALL)
soil = [
dict(date=row['date'],
soil_type=row['soil_type'],
geometry=row['geometry']) for row in csv_reader
]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.bulk_insert_mappings(mapper=SoilObject,
mappings=soil,
render_nulls=True,
return_defaults=False)
session.commit()
session.close()
def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
csv_reader = csv.DictReader(
f, delimiter=',',
quoting=csv.QUOTE_NONE) # quote non to skip whitespace
great_tits = [
dict(date=row['date'],
count=row['count'],
geometry=row['geometry']) for row in csv_reader
]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.bulk_insert_mappings(mapper=GreatTitObject,
mappings=great_tits,
render_nulls=True,
return_defaults=False)
session.commit()
session.close()
def load(self, transform_directory):
file_path = transform_directory / final_transformation_file(
transform_directory=transform_directory)
with open(file_path) as f:
csv_reader = csv.DictReader(
f, delimiter=',') # quote non to skip whitespace
weather_station_locations = [
dict(id=row['STN'],
name=row['NAME'],
geometry=Point(float(row['LON(east)']),
float(row['LAT(north)'])).wkt)
for row in csv_reader
]
session = sessionmaker(bind=SQLALCHEMY_ENGINE)()
session.bulk_insert_mappings(mapper=WeatherStationLocationObject,
mappings=weather_station_locations,
render_nulls=True,
return_defaults=False)
session.commit()
session.close()
