from pyspark.sql import SparkSession from pyspark.dbutils import DBUtils spark = SparkSession.builder.enableHiveSupport().getOrCreate() spark.sparkContext.setLogLevel("OFF") dbutils = DBUtils(spark) class util: def __init__(self, database, tables): self._database = database self._tables = tables self._options = "('nullValue'='', 'header'='True')" def create(self): self._createDatabase() self._createTables() def _createDatabase(self): spark.sql(f"CREATE DATABASE IF NOT EXISTS {self._database}") def _createTables(self): for table in self._tables: tableName = table["Name"] tablePath = table["Path"] dbCreate = f"DROP TABLE IF EXISTS {self._database}.{tableName}" print(dbCreate) spark.sql(dbCreate) tableCreate = f"CREATE TABLE {self._database}.{tableName} USING CSV OPTIONS {self._options} LOCATION '{tablePath}'" print(tableCreate) spark.sql(tableCreate)