""" Importiert die Backups aus dem Speicher und uploaded tageweise in amazon s3 """ import os import re import pandas as pd from datetime import datetime # compatibility with ipython # os.chdir(os.path.dirname(__file__)) from db import DatabaseWrapper import json import boto3 from pathlib import Path db = DatabaseWrapper("localhost", "hafasdb2", "hafas", "123") path = "/home/bemootzer/Documents/SoftwareProjekte/stewardless/stewardless-crawler/dbbackup" # path = "/home/bemootzer/Documents/SoftwareProjekte/stewardless/stewardless-crawler/dbbackup" re_sql = re.compile(r"arrival.*.sql") _, blacklist, _ = next(os.walk(os.path.join("summaries", "data"))) blacklist.append("2020-02-01") # data not complete for file in os.listdir(path): if re_sql.match(file): # LOAD BACKUP #file = "arrivals-2020-03-01.sql" date_string = file.replace("arrivals-", "").replace(".sql", "") date = datetime.fromisoformat(date_string) print(date) if date_string in blacklist or (date <= datetime(2020, 3, 22)):
import boto3 from pathlib import Path import geopandas.tools from shapely.geometry import Point import pymysql # download shapefiles countries = geopandas.GeoDataFrame.from_file( "https://raw.githubusercontent.com/AliceWi/TopoJSON-Germany/master/germany.json", layer=1, driver="TopoJSON") # clean unnecessary columns countries = countries[["id", "name", "geometry"]] countries.columns = ["landkreise_id", "landkreis", "geometry"] db = DatabaseWrapper("localhost", "hafasdb2", "hafas", "123") connection_aws = pymysql.connect(host=config["host"], user=config["user"], password=config["password"], db=config["database"], charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) connection_local = pymysql.connect(host="localhost", user="******", password="******", db="hafasdb2", charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor)