def get_form(cls, connector: 'MySQLConnector', current_config): """ Method to retrieve the form with a current config For example, once the connector is set, - we are able to give suggestions for the `database` field - if `database` is set, we are able to give suggestions for the `table` field """ constraints = {} with suppress(Exception): connection = pymysql.connect(**connector.get_connection_params( cursorclass=None, database=current_config.get('database'))) # Always add the suggestions for the available databases with connection.cursor() as cursor: cursor.execute('SHOW DATABASES;') res = cursor.fetchall() # res = (('information_schema',), ('mysql_db',)) available_dbs = [db_name for (db_name, ) in res] constraints['database'] = strlist_to_enum( 'database', available_dbs) if 'database' in current_config: cursor.execute('SHOW TABLES;') res = cursor.fetchall() available_tables = [table_name for (table_name, ) in res] constraints['table'] = strlist_to_enum( 'table', available_tables, None) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'MSSQLConnector', current_config): """ Method to retrieve the form with a current config For example, once the connector is set, - we are able to give suggestions for the `database` field - if `database` is set, we are able to give suggestions for the `table` field """ constraints = {} with suppress(Exception): connection = pyodbc.connect(**connector.get_connection_params( database=current_config.get('database', 'tempdb'))) # # Always add the suggestions for the available databases with connection.cursor() as cursor: cursor.execute('SELECT name FROM sys.databases;') res = cursor.fetchall() available_dbs = [r[0] for r in res] constraints['database'] = strlist_to_enum( 'database', available_dbs) if 'database' in current_config: cursor.execute( 'SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES;') res = cursor.fetchall() available_tables = [table_name for (table_name, ) in res] constraints['table'] = strlist_to_enum( 'table', available_tables, None) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'ClickhouseConnector', current_config): """ Method to retrieve the form with a current config For example, once the connector is set, - we are able to give suggestions for the `database` field - if `database` is set, we are able to give suggestions for the `table` field """ constraints = {} with suppress(Exception): connection = clickhouse_driver.connect( connector.get_connection_url()) # Always add the suggestions for the available databases with connection.cursor() as cursor: cursor.execute('SHOW DATABASES') res = cursor.fetchall() available_dbs = [ db_name for (db_name, ) in res if db_name != 'system' ] constraints['database'] = strlist_to_enum( 'database', available_dbs) if 'database' in current_config: cursor.execute( f"""SELECT name FROM system.tables WHERE database = '{current_config["database"]}'""" ) res = cursor.fetchall() available_tables = [table[0] for table in res] constraints['table'] = strlist_to_enum( 'table', available_tables, None) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'MySQLConnector', current_config): """ Method to retrieve the form with a current config For example, once the connector is set, - we are able to give suggestions for the `database` field - if `database` is set, we are able to give suggestions for the `table` field """ connection = pymysql.connect(**connector.get_connection_params( cursorclass=None, database=current_config.get('database'))) # Add constraints to the schema # the key has to be a valid field # the value is either <default value> or a tuple ( <type>, <default value> ) # If the field is required, the <default value> has to be '...' (cf pydantic doc) constraints = {} # # Always add the suggestions for the available databases with connection.cursor() as cursor: cursor.execute('SHOW DATABASES;') res = cursor.fetchall() # res = (('information_schema',), ('mysql_db',)) available_dbs = [db_name for (db_name, ) in res] constraints['database'] = strlist_to_enum('database', available_dbs) if 'database' in current_config: cursor.execute('SHOW TABLES;') res = cursor.fetchall() available_tables = [table_name for (table_name, ) in res] constraints['table'] = strlist_to_enum('table', available_tables) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'PostgresConnector', current_config): """ Method to retrieve the form with a current config For example, once the connector is set, - we are able to give suggestions for the `database` field - if `database` is set, we are able to give suggestions for the `table` field """ constraints = {} with suppress(Exception): connection = pgsql.connect( **connector.get_connection_params( database=current_config.get('database', 'postgres') ) ) # # Always add the suggestions for the available databases with connection.cursor() as cursor: cursor.execute("""select datname from pg_database where datistemplate = false;""") res = cursor.fetchall() available_dbs = [db_name for (db_name,) in res] constraints['database'] = strlist_to_enum('database', available_dbs) if 'database' in current_config: cursor.execute( """select table_schema, table_name from information_schema.tables where table_schema NOT IN ('pg_catalog', 'information_schema');""" ) res = cursor.fetchall() available_tables = [table_name for (_, table_name) in res] constraints['table'] = strlist_to_enum('table', available_tables) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'MongoConnector', current_config): """ Method to retrieve the form with a current config For example, once the connector is set, - we are able to give suggestions for the `database` field - if `database` is set, we are able to give suggestions for the `collection` field """ client = pymongo.MongoClient(connector.uri, ssl=connector.ssl) # Add constraints to the schema # the key has to be a valid field # the value is either <default value> or a tuple ( <type>, <default value> ) # If the field is required, the <default value> has to be '...' (cf pydantic doc) constraints = {} # Always add the suggestions for the available databases available_databases = client.list_database_names() constraints['database'] = strlist_to_enum('database', available_databases) if 'database' in current_config: validate_database(client, current_config['database']) available_cols = client[ current_config['database']].list_collection_names() constraints['collection'] = strlist_to_enum( 'collection', available_cols) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'SnowflakeConnector', current_config): constraints = {} with suppress(Exception): databases = connector._get_databases() warehouses = connector._get_warehouses() # Restrict some fields to lists of existing counterparts constraints['database'] = strlist_to_enum('database', databases) constraints['warehouse'] = strlist_to_enum('warehouse', warehouses) res = create_model('FormSchema', **constraints, __base__=cls).schema() res['properties']['warehouse']['default'] = connector.default_warehouse return res
def test_strlist_to_enum_default_value(): """It should be possible to add a default value (not required)""" model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'], 'pika')) assert model.schema() == { 'title': 'Test', 'type': 'object', 'definitions': { 'pokemon': { 'description': 'An enumeration.', 'enum': ['pika', 'bulbi'], 'title': 'pokemon', 'type': 'string', } }, 'properties': { 'pokemon': { 'allOf': [{ '$ref': '#/definitions/pokemon' }], 'default': 'pika' } }, }
def get_form(cls, connector: 'GithubConnector', current_config, **kwargs): """Retrieve a form filled with suggestions of available organizations.""" # Always add the suggestions for the available organizations constraints = {} with suppress(Exception): available_organizations = connector.get_organizations() constraints['organization'] = strlist_to_enum( 'organization', available_organizations) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'GoogleSheetsConnector', current_config, **kwargs): """Retrieve a form filled with suggestions of available sheets.""" # Always add the suggestions for the available sheets constraints = {} with suppress(Exception): available_sheets = connector.list_sheets( current_config['spreadsheet_id']) constraints['sheet'] = strlist_to_enum('sheet', available_sheets) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'MongoConnector', current_config): """ Method to retrieve the form with a current config For example, once the connector is set, - we are able to give suggestions for the `database` field - if `database` is set, we are able to give suggestions for the `collection` field """ constraints = {} client = pymongo.MongoClient(**connector._get_mongo_client_kwargs()) # Always add the suggestions for the available databases available_databases = client.list_database_names() constraints['database'] = strlist_to_enum('database', available_databases) if 'database' in current_config: validate_database(client, current_config['database']) available_cols = client[current_config['database']].list_collection_names() constraints['collection'] = strlist_to_enum('collection', available_cols) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'SoapConnector', current_config): constraints = {} client = connector.create_client() methods_docs = cls._get_methods_docs(client) constraints['method'] = strlist_to_enum('method', list(methods_docs.keys())) res = create_model('FormSchema', **constraints, __base__=cls).schema() res['properties']['parameters'][ 'description'] = f'Services documentation: <br> {"<br>".join(list(methods_docs.values()))}' return res
def get_form(cls, connector: 'RedshiftConnector', current_config): constraints = {} with suppress(Exception): if 'database' in current_config: ds = RedshiftDataSource(domain='Redshift', name='redshift', database=current_config['database']) available_tables = connector._retrieve_tables( database=ds.database) constraints['table'] = strlist_to_enum('table', available_tables, None) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'GoogleSheetsConnector', current_config): # Always add the suggestions for the available sheets constraints = {} with suppress(Exception): data = connector.bearer_oauth_get_endpoint( current_config['spreadsheet_id']) available_sheets = [ str(x['properties']['title']) for x in data['sheets'] ] constraints['sheet'] = strlist_to_enum('sheet', available_sheets) return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'GoogleSheets2Connector', current_config, **kwargs): """Retrieve a form filled with suggestions of available sheets.""" # Always add the suggestions for the available sheets constraints = {} with suppress(Exception): partial_endpoint = current_config['spreadsheet_id'] final_url = f'{connector._baseroute}{partial_endpoint}' data = connector._run_fetch(final_url) available_sheets = [ str(x['properties']['title']) for x in data['sheets'] ] constraints['sheet'] = strlist_to_enum('sheet', available_sheets) return create_model('FormSchema', **constraints, __base__=cls).schema()
def test_strlist_to_enum_required(): """It should be required by default""" model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'])) assert model.schema() == { 'title': 'Test', 'type': 'object', 'properties': { 'pokemon': { 'title': 'Pokemon', 'enum': ['pika', 'bulbi'], 'type': 'string' } }, 'required': ['pokemon'] }
def test_strlist_to_enum_required(): """It should be required by default""" model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'])) assert model.schema() == { 'title': 'Test', 'type': 'object', 'definitions': { 'pokemon': { 'description': 'An enumeration.', 'enum': ['pika', 'bulbi'], 'title': 'pokemon', 'type': 'string', } }, 'properties': {'pokemon': {'$ref': '#/definitions/pokemon'}}, 'required': ['pokemon'], }
def test_strlist_to_enum_default_value(): """It should be possible to add a default value (not required)""" model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'], 'pika')) assert model.schema() == { 'title': 'Test', 'type': 'object', 'properties': { 'pokemon': { 'title': 'Pokemon', 'default': 'pika', 'enum': ['pika', 'bulbi'], 'type': 'string' } } }
def get_form(cls, connector: 'OracleSQLConnector', current_config): """ Method to retrieve the form with a current config For example, once the connector is set, - we are able to give suggestions for the `database` field - if `database` is set, we are able to give suggestions for the `table` field """ constraints = {} with suppress(Exception): connection = cx_Oracle.connect(**connector.get_connection_params()) with connection.cursor() as cursor: cursor.execute("""select table_name from ALL_TABLES""") res = cursor.fetchall() # Filter tables starting with an '_' because strlist_to_enum cannot # set attributes starting with '_' available_tables = [table_name for (table_name,) in res if table_name[0] != '_'] constraints['table'] = strlist_to_enum('table', available_tables, None) return create_model('FormSchema', **constraints, __base__=cls).schema()