""" Importiert die Backups aus dem Speicher und
uploaded tageweise in amazon s3 """
import os
import re
import pandas as pd
from datetime import datetime
# compatibility with ipython
# os.chdir(os.path.dirname(__file__))
from db import DatabaseWrapper
import json
import boto3
from pathlib import Path
db = DatabaseWrapper("localhost", "hafasdb2", "hafas", "123")
path = "/home/bemootzer/Documents/SoftwareProjekte/stewardless/stewardless-crawler/dbbackup"
# path = "/home/bemootzer/Documents/SoftwareProjekte/stewardless/stewardless-crawler/dbbackup"
re_sql = re.compile(r"arrival.*.sql")
_, blacklist, _ = next(os.walk(os.path.join("summaries", "data")))
blacklist.append("2020-02-01") # data not complete
for file in os.listdir(path):
if re_sql.match(file):
# LOAD BACKUP
#file = "arrivals-2020-03-01.sql"
date_string = file.replace("arrivals-", "").replace(".sql", "")
date = datetime.fromisoformat(date_string)
print(date)
if date_string in blacklist or (date <= datetime(2020, 3, 22)):
import boto3
from pathlib import Path
import geopandas.tools
from shapely.geometry import Point
import pymysql
# download shapefiles
countries = geopandas.GeoDataFrame.from_file(
"https://raw.githubusercontent.com/AliceWi/TopoJSON-Germany/master/germany.json",
layer=1,
driver="TopoJSON")
# clean unnecessary columns
countries = countries[["id", "name", "geometry"]]
countries.columns = ["landkreise_id", "landkreis", "geometry"]
db = DatabaseWrapper("localhost", "hafasdb2", "hafas", "123")
connection_aws = pymysql.connect(host=config["host"],
user=config["user"],
password=config["password"],
db=config["database"],
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
connection_local = pymysql.connect(host="localhost",
user="******",
password="******",
db="hafasdb2",
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
