def get_form(cls, connector: 'MySQLConnector', current_config):
"""
Method to retrieve the form with a current config
For example, once the connector is set,
- we are able to give suggestions for the `database` field
- if `database` is set, we are able to give suggestions for the `table` field
"""
constraints = {}
with suppress(Exception):
connection = pymysql.connect(**connector.get_connection_params(
cursorclass=None, database=current_config.get('database')))
# Always add the suggestions for the available databases
with connection.cursor() as cursor:
cursor.execute('SHOW DATABASES;')
res = cursor.fetchall()
# res = (('information_schema',), ('mysql_db',))
available_dbs = [db_name for (db_name, ) in res]
constraints['database'] = strlist_to_enum(
'database', available_dbs)
if 'database' in current_config:
cursor.execute('SHOW TABLES;')
res = cursor.fetchall()
available_tables = [table_name for (table_name, ) in res]
constraints['table'] = strlist_to_enum(
'table', available_tables, None)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'MSSQLConnector', current_config):
"""
Method to retrieve the form with a current config
For example, once the connector is set,
- we are able to give suggestions for the `database` field
- if `database` is set, we are able to give suggestions for the `table` field
"""
constraints = {}
with suppress(Exception):
connection = pyodbc.connect(**connector.get_connection_params(
database=current_config.get('database', 'tempdb')))
# # Always add the suggestions for the available databases
with connection.cursor() as cursor:
cursor.execute('SELECT name FROM sys.databases;')
res = cursor.fetchall()
available_dbs = [r[0] for r in res]
constraints['database'] = strlist_to_enum(
'database', available_dbs)
if 'database' in current_config:
cursor.execute(
'SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES;')
res = cursor.fetchall()
available_tables = [table_name for (table_name, ) in res]
constraints['table'] = strlist_to_enum(
'table', available_tables, None)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'ClickhouseConnector', current_config):
"""
Method to retrieve the form with a current config
For example, once the connector is set,
- we are able to give suggestions for the `database` field
- if `database` is set, we are able to give suggestions for the `table` field
"""
constraints = {}
with suppress(Exception):
connection = clickhouse_driver.connect(
connector.get_connection_url())
# Always add the suggestions for the available databases
with connection.cursor() as cursor:
cursor.execute('SHOW DATABASES')
res = cursor.fetchall()
available_dbs = [
db_name for (db_name, ) in res if db_name != 'system'
]
constraints['database'] = strlist_to_enum(
'database', available_dbs)
if 'database' in current_config:
cursor.execute(
f"""SELECT name FROM system.tables WHERE database = '{current_config["database"]}'"""
)
res = cursor.fetchall()
available_tables = [table[0] for table in res]
constraints['table'] = strlist_to_enum(
'table', available_tables, None)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'MySQLConnector', current_config):
"""
Method to retrieve the form with a current config
For example, once the connector is set,
- we are able to give suggestions for the `database` field
- if `database` is set, we are able to give suggestions for the `table` field
"""
connection = pymysql.connect(**connector.get_connection_params(
cursorclass=None, database=current_config.get('database')))
# Add constraints to the schema
# the key has to be a valid field
# the value is either <default value> or a tuple ( <type>, <default value> )
# If the field is required, the <default value> has to be '...' (cf pydantic doc)
constraints = {}
# # Always add the suggestions for the available databases
with connection.cursor() as cursor:
cursor.execute('SHOW DATABASES;')
res = cursor.fetchall()
# res = (('information_schema',), ('mysql_db',))
available_dbs = [db_name for (db_name, ) in res]
constraints['database'] = strlist_to_enum('database',
available_dbs)
if 'database' in current_config:
cursor.execute('SHOW TABLES;')
res = cursor.fetchall()
available_tables = [table_name for (table_name, ) in res]
constraints['table'] = strlist_to_enum('table',
available_tables)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'PostgresConnector', current_config):
"""
Method to retrieve the form with a current config
For example, once the connector is set,
- we are able to give suggestions for the `database` field
- if `database` is set, we are able to give suggestions for the `table` field
"""
constraints = {}
with suppress(Exception):
connection = pgsql.connect(
**connector.get_connection_params(
database=current_config.get('database', 'postgres')
)
)
# # Always add the suggestions for the available databases
with connection.cursor() as cursor:
cursor.execute("""select datname from pg_database where datistemplate = false;""")
res = cursor.fetchall()
available_dbs = [db_name for (db_name,) in res]
constraints['database'] = strlist_to_enum('database', available_dbs)
if 'database' in current_config:
cursor.execute(
"""select table_schema, table_name from information_schema.tables
where table_schema NOT IN ('pg_catalog', 'information_schema');"""
)
res = cursor.fetchall()
available_tables = [table_name for (_, table_name) in res]
constraints['table'] = strlist_to_enum('table', available_tables)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'MongoConnector', current_config):
"""
Method to retrieve the form with a current config
For example, once the connector is set,
- we are able to give suggestions for the `database` field
- if `database` is set, we are able to give suggestions for the `collection` field
"""
client = pymongo.MongoClient(connector.uri, ssl=connector.ssl)
# Add constraints to the schema
# the key has to be a valid field
# the value is either <default value> or a tuple ( <type>, <default value> )
# If the field is required, the <default value> has to be '...' (cf pydantic doc)
constraints = {}
# Always add the suggestions for the available databases
available_databases = client.list_database_names()
constraints['database'] = strlist_to_enum('database',
available_databases)
if 'database' in current_config:
validate_database(client, current_config['database'])
available_cols = client[
current_config['database']].list_collection_names()
constraints['collection'] = strlist_to_enum(
'collection', available_cols)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'SnowflakeConnector', current_config):
constraints = {}
with suppress(Exception):
databases = connector._get_databases()
warehouses = connector._get_warehouses()
# Restrict some fields to lists of existing counterparts
constraints['database'] = strlist_to_enum('database', databases)
constraints['warehouse'] = strlist_to_enum('warehouse', warehouses)
res = create_model('FormSchema', **constraints, __base__=cls).schema()
res['properties']['warehouse']['default'] = connector.default_warehouse
return res
def test_strlist_to_enum_default_value():
"""It should be possible to add a default value (not required)"""
model = create_model('Test',
pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'],
'pika'))
assert model.schema() == {
'title': 'Test',
'type': 'object',
'definitions': {
'pokemon': {
'description': 'An enumeration.',
'enum': ['pika', 'bulbi'],
'title': 'pokemon',
'type': 'string',
}
},
'properties': {
'pokemon': {
'allOf': [{
'$ref': '#/definitions/pokemon'
}],
'default': 'pika'
}
},
}
def get_form(cls, connector: 'GithubConnector', current_config, **kwargs):
"""Retrieve a form filled with suggestions of available organizations."""
# Always add the suggestions for the available organizations
constraints = {}
with suppress(Exception):
available_organizations = connector.get_organizations()
constraints['organization'] = strlist_to_enum(
'organization', available_organizations)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'GoogleSheetsConnector', current_config,
**kwargs):
"""Retrieve a form filled with suggestions of available sheets."""
# Always add the suggestions for the available sheets
constraints = {}
with suppress(Exception):
available_sheets = connector.list_sheets(
current_config['spreadsheet_id'])
constraints['sheet'] = strlist_to_enum('sheet', available_sheets)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'MongoConnector', current_config):
"""
Method to retrieve the form with a current config
For example, once the connector is set,
- we are able to give suggestions for the `database` field
- if `database` is set, we are able to give suggestions for the `collection` field
"""
constraints = {}
client = pymongo.MongoClient(**connector._get_mongo_client_kwargs())
# Always add the suggestions for the available databases
available_databases = client.list_database_names()
constraints['database'] = strlist_to_enum('database', available_databases)
if 'database' in current_config:
validate_database(client, current_config['database'])
available_cols = client[current_config['database']].list_collection_names()
constraints['collection'] = strlist_to_enum('collection', available_cols)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'SoapConnector', current_config):
constraints = {}
client = connector.create_client()
methods_docs = cls._get_methods_docs(client)
constraints['method'] = strlist_to_enum('method',
list(methods_docs.keys()))
res = create_model('FormSchema', **constraints, __base__=cls).schema()
res['properties']['parameters'][
'description'] = f'Services documentation: <br> {"<br>".join(list(methods_docs.values()))}'
return res
def get_form(cls, connector: 'RedshiftConnector', current_config):
constraints = {}
with suppress(Exception):
if 'database' in current_config:
ds = RedshiftDataSource(domain='Redshift',
name='redshift',
database=current_config['database'])
available_tables = connector._retrieve_tables(
database=ds.database)
constraints['table'] = strlist_to_enum('table',
available_tables, None)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'GoogleSheetsConnector', current_config):
# Always add the suggestions for the available sheets
constraints = {}
with suppress(Exception):
data = connector.bearer_oauth_get_endpoint(
current_config['spreadsheet_id'])
available_sheets = [
str(x['properties']['title']) for x in data['sheets']
]
constraints['sheet'] = strlist_to_enum('sheet', available_sheets)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def get_form(cls, connector: 'GoogleSheets2Connector', current_config,
**kwargs):
"""Retrieve a form filled with suggestions of available sheets."""
# Always add the suggestions for the available sheets
constraints = {}
with suppress(Exception):
partial_endpoint = current_config['spreadsheet_id']
final_url = f'{connector._baseroute}{partial_endpoint}'
data = connector._run_fetch(final_url)
available_sheets = [
str(x['properties']['title']) for x in data['sheets']
]
constraints['sheet'] = strlist_to_enum('sheet', available_sheets)
return create_model('FormSchema', **constraints, __base__=cls).schema()
def test_strlist_to_enum_required():
"""It should be required by default"""
model = create_model('Test',
pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi']))
assert model.schema() == {
'title': 'Test',
'type': 'object',
'properties': {
'pokemon': {
'title': 'Pokemon',
'enum': ['pika', 'bulbi'],
'type': 'string'
}
},
'required': ['pokemon']
}
def test_strlist_to_enum_required():
"""It should be required by default"""
model = create_model('Test', pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi']))
assert model.schema() == {
'title': 'Test',
'type': 'object',
'definitions': {
'pokemon': {
'description': 'An enumeration.',
'enum': ['pika', 'bulbi'],
'title': 'pokemon',
'type': 'string',
}
},
'properties': {'pokemon': {'$ref': '#/definitions/pokemon'}},
'required': ['pokemon'],
}
def test_strlist_to_enum_default_value():
"""It should be possible to add a default value (not required)"""
model = create_model('Test',
pokemon=strlist_to_enum('pokemon', ['pika', 'bulbi'],
'pika'))
assert model.schema() == {
'title': 'Test',
'type': 'object',
'properties': {
'pokemon': {
'title': 'Pokemon',
'default': 'pika',
'enum': ['pika', 'bulbi'],
'type': 'string'
}
}
}
def get_form(cls, connector: 'OracleSQLConnector', current_config):
"""
Method to retrieve the form with a current config
For example, once the connector is set,
- we are able to give suggestions for the `database` field
- if `database` is set, we are able to give suggestions for the `table` field
"""
constraints = {}
with suppress(Exception):
connection = cx_Oracle.connect(**connector.get_connection_params())
with connection.cursor() as cursor:
cursor.execute("""select table_name from ALL_TABLES""")
res = cursor.fetchall()
# Filter tables starting with an '_' because strlist_to_enum cannot
# set attributes starting with '_'
available_tables = [table_name for (table_name,) in res if table_name[0] != '_']
constraints['table'] = strlist_to_enum('table', available_tables, None)
return create_model('FormSchema', **constraints, __base__=cls).schema()
