def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: csv_reader = csv.DictReader(f, delimiter=',', quoting=csv.QUOTE_ALL) townships_interpolated = [{ "code": row['id'], "name": row['name'], "township": row['township'], "year": datetime.strptime(row['year'], "%Y-%m-%d %H:%M:%S").year, self.interpolated_value_name: Decimal(row['interpolated_values']) } for row in csv_reader] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.bulk_insert_mappings(mapper=self.model, mappings=townships_interpolated, render_nulls=True, return_defaults=False) session.commit() session.close()
def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: csv_reader = csv.DictReader( f, delimiter=',', quoting=csv.QUOTE_NONE) # quote non to skip whitespace weather_station_data = [ dict(station_id=int(row['station_id']), date=datetime.strptime(row['date'], "%Y-%m-%d"), temperature_avg=Decimal(row['temperature_avg']), temperature_min=Decimal(row['temperature_min']), temperature_max=Decimal(row['temperature_max']), sunshine_duration=Decimal(row['sunshine_duration']), sunshine_radiation=Decimal(row['sunshine_radiation']), rain_duration=Decimal(row['rain_duration']), rain_sum=Decimal(row['rain_sum']), humidity_avg=Decimal(row['humidity_avg']), humidity_max=Decimal(row['humidity_max']), humidity_min=Decimal(row['humidity_min'])) for row in csv_reader ] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.bulk_insert_mappings(mapper=WeatherStationDataObject, mappings=weather_station_data, render_nulls=True, return_defaults=False) session.commit() session.close()
def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: csv_reader = csv.DictReader( f, delimiter=',', quoting=csv.QUOTE_NONE) # quote non to skip whitespace trees = [ dict( # species_latin=row['species_latin'], species_dutch=row['species_dutch'], geometry=row['geometry'], origin='amsterdam') for row in csv_reader ] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.bulk_insert_mappings(mapper=TreeObject, mappings=trees, render_nulls=True, return_defaults=False) session.commit() session.close()
def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: csv_reader = csv.DictReader( f, delimiter=',', quoting=csv.QUOTE_NONE) # quote non to skip whitespace oak_processionary_moths = [ dict(date=row['date'], stage=row['stage'], geometry=row['geometry'], origin='vlinderstichting', granularity='moth') for row in csv_reader ] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.bulk_insert_mappings(mapper=OakProcessionaryMothObject, mappings=oak_processionary_moths, render_nulls=True, return_defaults=False) session.commit() session.close()
def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: csv_reader = csv.DictReader(f, delimiter=',', quoting=csv.QUOTE_ALL) neighbourhoods = [ dict(code=row['id'], name=row['name'], township=row['township'], geometry=row['geometry'], area=float(row['area'])) for row in csv_reader ] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.bulk_insert_mappings(mapper=NeighbourhoodObject, mappings=neighbourhoods, render_nulls=True, return_defaults=False) session.commit() session.close()
def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: json_file = json.load(f) townships = [ TownshipObject(name=line['properties']['name'], code=line['properties']['code'], geometry=shape(line['geometry']).wkt) for line in json_file['features'] ] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.add_all(townships) session.commit() session.close()
def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: csv_reader = csv.DictReader(f, delimiter=',', quoting=csv.QUOTE_ALL) soil = [ dict(date=row['date'], soil_type=row['soil_type'], geometry=row['geometry']) for row in csv_reader ] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.bulk_insert_mappings(mapper=SoilObject, mappings=soil, render_nulls=True, return_defaults=False) session.commit() session.close()
def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: csv_reader = csv.DictReader( f, delimiter=',', quoting=csv.QUOTE_NONE) # quote non to skip whitespace great_tits = [ dict(date=row['date'], count=row['count'], geometry=row['geometry']) for row in csv_reader ] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.bulk_insert_mappings(mapper=GreatTitObject, mappings=great_tits, render_nulls=True, return_defaults=False) session.commit() session.close()
def load(self, transform_directory): file_path = transform_directory / final_transformation_file( transform_directory=transform_directory) with open(file_path) as f: csv_reader = csv.DictReader( f, delimiter=',') # quote non to skip whitespace weather_station_locations = [ dict(id=row['STN'], name=row['NAME'], geometry=Point(float(row['LON(east)']), float(row['LAT(north)'])).wkt) for row in csv_reader ] session = sessionmaker(bind=SQLALCHEMY_ENGINE)() session.bulk_insert_mappings(mapper=WeatherStationLocationObject, mappings=weather_station_locations, render_nulls=True, return_defaults=False) session.commit() session.close()