def __init__(self,
                 name="default",
                 data_context=None,
                 data_asset_type=None,
                 credentials=None,
                 batch_kwargs_generators=None,
                 **kwargs):
        if not sqlalchemy:
            raise DatasourceInitializationError(
                name, "ModuleNotFoundError: No module named 'sqlalchemy'")

        configuration_with_defaults = SqlAlchemyDatasource.build_configuration(
            data_asset_type, batch_kwargs_generators, **kwargs)
        data_asset_type = configuration_with_defaults.pop("data_asset_type")
        batch_kwargs_generators = configuration_with_defaults.pop(
            "batch_kwargs_generators", None)
        super(SqlAlchemyDatasource,
              self).__init__(name,
                             data_context=data_context,
                             data_asset_type=data_asset_type,
                             batch_kwargs_generators=batch_kwargs_generators,
                             **configuration_with_defaults)

        if credentials is not None:
            self._datasource_config.update({"credentials": credentials})
        else:
            credentials = {}

        try:
            # if an engine was provided, use that
            if "engine" in kwargs:
                self.engine = kwargs.pop("engine")

            # if a connection string or url was provided, use that
            elif "connection_string" in kwargs:
                connection_string = kwargs.pop("connection_string")
                self.engine = create_engine(connection_string, **kwargs)
                self.engine.connect()
            elif "url" in credentials:
                url = credentials.pop("url")
                # TODO perhaps we could carefully regex out the driver from the
                #  url. It would need to be cautious to avoid leaking secrets.
                self.drivername = "other"
                self.engine = create_engine(url, **kwargs)
                self.engine.connect()

            # Otherwise, connect using remaining kwargs
            else:
                options, drivername = self._get_sqlalchemy_connection_options(
                    **kwargs)
                self.drivername = drivername
                self.engine = create_engine(options)
                self.engine.connect()

        except (sqlalchemy.exc.OperationalError,
                sqlalchemy.exc.DatabaseError) as sqlalchemy_error:
            raise DatasourceInitializationError(self._name,
                                                str(sqlalchemy_error))

        self._build_generators()
Exemplo n.º 2
0
    def __init__(self,
                 name="default",
                 data_context=None,
                 data_asset_type=None,
                 profile=None,
                 generators=None,
                 **kwargs):
        if not sqlalchemy:
            raise DatasourceInitializationError(
                name, "ModuleNotFoundError: No module named 'sqlalchemy'")

        if generators is None:
            generators = {"default": {"type": "queries"}}

        if data_asset_type is None:
            data_asset_type = ClassConfig(class_name="SqlAlchemyDataset")
        else:
            try:
                data_asset_type = ClassConfig(**data_asset_type)
            except TypeError:
                # In this case, we allow the passed config, for now, in case they're using a legacy string-only config
                pass

        super(SqlAlchemyDatasource,
              self).__init__(name,
                             type_="sqlalchemy",
                             data_context=data_context,
                             data_asset_type=data_asset_type,
                             generators=generators)
        if profile is not None:
            self._datasource_config.update({"profile": profile})

        try:
            # if an engine was provided, use that
            if "engine" in kwargs:
                self.engine = kwargs.pop("engine")

            # if a connection string or url was provided, use that
            elif "connection_string" in kwargs:
                connection_string = kwargs.pop("connection_string")
                self.engine = create_engine(connection_string, **kwargs)
                self.engine.connect()
            elif "url" in kwargs:
                url = kwargs.pop("url")
                self.engine = create_engine(url, **kwargs)
                self.engine.connect()

            # Otherwise, connect using remaining kwargs
            else:
                self.engine = create_engine(
                    self._get_sqlalchemy_connection_options(**kwargs))
                self.engine.connect()

        except sqlalchemy.exc.OperationalError as sqlalchemy_error:
            raise DatasourceInitializationError(self._name,
                                                str(sqlalchemy_error))

        self._build_generators()
Exemplo n.º 3
0
    def __init__(self,
                 name="default",
                 data_context=None,
                 data_asset_type=None,
                 credentials=None,
                 generators=None,
                 **kwargs):
        if not sqlalchemy:
            raise DatasourceInitializationError(
                name, "ModuleNotFoundError: No module named 'sqlalchemy'")

        configuration_with_defaults = SqlAlchemyDatasource.build_configuration(
            data_asset_type, generators, **kwargs)
        data_asset_type = configuration_with_defaults.pop("data_asset_type")
        generators = configuration_with_defaults.pop("generators")
        super(SqlAlchemyDatasource,
              self).__init__(name,
                             data_context=data_context,
                             data_asset_type=data_asset_type,
                             generators=generators,
                             **configuration_with_defaults)

        if credentials is not None:
            self._datasource_config.update({"credentials": credentials})
        else:
            credentials = {}

        try:
            # if an engine was provided, use that
            if "engine" in kwargs:
                self.engine = kwargs.pop("engine")

            # if a connection string or url was provided, use that
            elif "connection_string" in kwargs:
                connection_string = kwargs.pop("connection_string")
                self.engine = create_engine(connection_string, **kwargs)
                self.engine.connect()
            elif "url" in credentials:
                url = credentials.pop("url")
                self.engine = create_engine(url, **kwargs)
                self.engine.connect()

            # Otherwise, connect using remaining kwargs
            else:
                self.engine = create_engine(
                    self._get_sqlalchemy_connection_options(**kwargs))
                self.engine.connect()

        except sqlalchemy.exc.OperationalError as sqlalchemy_error:
            raise DatasourceInitializationError(self._name,
                                                str(sqlalchemy_error))

        self._build_generators()
    def __init__(self,
                 name="default",
                 data_context=None,
                 data_asset_type=None,
                 credentials=None,
                 batch_kwargs_generators=None,
                 **kwargs):
        if not sqlalchemy:
            raise DatasourceInitializationError(
                name, "ModuleNotFoundError: No module named 'sqlalchemy'")

        configuration_with_defaults = SqlAlchemyDatasource.build_configuration(
            data_asset_type, batch_kwargs_generators, **kwargs)
        data_asset_type = configuration_with_defaults.pop("data_asset_type")
        batch_kwargs_generators = configuration_with_defaults.pop(
            "batch_kwargs_generators", None)
        super().__init__(name,
                         data_context=data_context,
                         data_asset_type=data_asset_type,
                         batch_kwargs_generators=batch_kwargs_generators,
                         **configuration_with_defaults)

        if credentials is not None:
            self._datasource_config.update({"credentials": credentials})
        else:
            credentials = {}

        try:
            # if an engine was provided, use that
            if "engine" in kwargs:
                self.engine = kwargs.pop("engine")

            # if a connection string or url was provided, use that
            elif "connection_string" in kwargs:
                connection_string = kwargs.pop("connection_string")
                self.engine = create_engine(connection_string, **kwargs)
                connection = self.engine.connect()
                connection.close()
            elif "url" in credentials:
                url = credentials.pop("url")
                self.drivername = urlparse(url).scheme
                self.engine = create_engine(url, **kwargs)
                connection = self.engine.connect()
                connection.close()

            # Otherwise, connect using remaining kwargs
            else:
                (
                    options,
                    create_engine_kwargs,
                    drivername,
                ) = self._get_sqlalchemy_connection_options(**kwargs)
                self.drivername = drivername
                self.engine = create_engine(options, **create_engine_kwargs)
                connection = self.engine.connect()
                connection.close()

            # since we switched to lazy loading of Datasources when we initialise a DataContext,
            # the dialect of SQLAlchemy Datasources cannot be obtained reliably when we send
            # "data_context.__init__" events.
            # This event fills in the SQLAlchemy dialect.
            if data_context is not None and getattr(
                    data_context, "_usage_statistics_handler", None):
                handler = data_context._usage_statistics_handler
                handler.send_usage_message(
                    event="datasource.sqlalchemy.connect",
                    event_payload={
                        "anonymized_name":
                        handler._datasource_anonymizer.anonymize(self.name),
                        "sqlalchemy_dialect":
                        self.engine.name,
                    },
                    success=True,
                )

        except datasource_initialization_exceptions as sqlalchemy_error:
            raise DatasourceInitializationError(self._name,
                                                str(sqlalchemy_error))

        self._build_generators()