def setUpClass(cls):
override_config = {
'integrations': {},
'api': {
"http": {
"host": get_docker0_inet_ip()
},
"mysql": {
"host": get_docker0_inet_ip()
}
}
}
run_environment(apis=['http', 'mysql'],
override_config=override_config)
cls.docker_client = docker.from_env()
cls.mysql_image = 'mysql'
cls.config = json.loads(Path(CONFIG_PATH).read_text())
with open(EXTERNAL_DB_CREDENTIALS, 'rt') as f:
cls.db_creds = json.load(f)
cls.launch_query_tmpl = "mysql --host=%s --port=%s --user=%s --database=mindsdb" % (
cls.config["api"]["mysql"]["host"],
cls.config["api"]["mysql"]["port"],
cls.config["api"]["mysql"]["user"])
def test_4_wrong_prediction(self):
'''
close mindsdb while make prediction, then try run it again
'''
ioloop = asyncio.get_event_loop()
if ioloop.is_closed():
ioloop = asyncio.new_event_loop()
ioloop.run_in_executor(
None, lambda: requests.post(
f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}_2/predict',
json={'when': CONDITION}))
time.sleep(0.5)
stop_mindsdb()
ioloop.close()
run_environment(apis=['mysql', 'http'],
override_config={
'integrations': {
'default_mariadb': {
'publish': True
}
}
})
res = requests.post(
f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}_2/predict',
json={'when': CONDITION})
assert res.status_code == 200
def setUpClass(cls):
run_environment(apis=['mongodb'],
override_config={
'integrations': {
'default_mongodb': {
'publish': True
}
},
'permanent_storage': {
'location': 'local'
}
})
config.update(json.loads(Path(CONFIG_PATH).read_text()))
connection = config['api']['mongodb']
kwargs = {}
if len(connection.get('password', '')) > 0:
kwargs['password'] = connection.get('password')
if len(connection.get('user', '')) > 0:
kwargs['username'] = connection.get('user')
cls.mongo_mindsdb = MongoClient(host=connection['host'],
port=int(connection.get('port',
47336)),
**kwargs)
integraton = config['integrations']['default_mongodb']
cls.mongo_server = MongoClient(host=integraton['host'],
port=integraton.get('port', 27017),
username=integraton['user'],
password=integraton['password'])
def setUpClass(cls):
run_environment(apis=['http'])
config.update(json.loads(Path(CONFIG_PATH).read_text()))
data = fetch(f'select * from test_data.{TEST_DATA_TABLE} limit 50',
as_dict=True)
cls.external_datasource_csv_path = make_test_csv(
EXTERNAL_DS_NAME, data)
def setUpClass(cls):
run_environment(
apis=['http', 'mongodb']
)
config.update(
json.loads(
Path(CONFIG_PATH).read_text()
)
)
def setUpClass(cls):
run_environment(apis=['mysql', 'http'],
override_config={
'integrations': {
INTEGRATION_NAME: {
'publish': True
}
}
})
config.update(json.loads(Path(CONFIG_PATH).read_text()))
def test_2_restart_and_connect(self):
stop_mindsdb()
run_environment(apis=['mysql'],
override_config={
'integrations': {
'default_mariadb': {
'publish': False
}
}
})
config.update(json.loads(Path(CONFIG_PATH).read_text()))
def test_3_wrong_predictor(self):
'''
try create predictor with wrong parameters,
close mindsdb while model training
check mindsdb can start again
'''
check_predictor_not_exists(TEST_PREDICTOR)
data = {
'to_predict': list(TO_PREDICT.keys()),
'data_source_name': 'wrong ds'
}
res = requests.put(f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}',
json=data)
assert 'Can not find datasource' in res.json()['message']
check_predictor_not_exists(TEST_PREDICTOR)
data = {
'to_predict': list(TO_PREDICT.keys()),
'data_source_name': TEST_DS
}
res = requests.put(f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}',
json=data)
assert res.status_code == 200
stop_mindsdb()
run_environment(apis=['mysql', 'http'],
override_config={
'integrations': {
'default_mariadb': {
'publish': True
}
}
})
# TODO add after this issue will be closed: https://github.com/mindsdb/mindsdb/issues/948
# check_predictor_not_exists(TEST_PREDICTOR)
data = {
'to_predict': list(TO_PREDICT.keys()),
'data_source_name': TEST_DS
}
res = requests.put(f'{HTTP_API_ROOT}/predictors/{TEST_PREDICTOR}_2',
json=data)
assert res.status_code == 200
wait_predictor_learn(f'{TEST_PREDICTOR}_2')
def setUpClass(cls):
mdb, datastore = run_environment('mysql', config)
cls.mdb = mdb
models = cls.mdb.get_models()
models = [x['name'] for x in models]
if TEST_PREDICTOR_NAME in models:
cls.mdb.delete_model(TEST_PREDICTOR_NAME)
test_csv_path = get_test_csv(TEST_CSV['name'], TEST_CSV['url'])
query('create database if not exists test')
test_tables = query('show tables from test')
test_tables = [x[0] for x in test_tables]
if TEST_DATA_TABLE not in test_tables:
print('creating test data table...')
query(f'''
CREATE TABLE test.{TEST_DATA_TABLE} (
number_of_rooms int,
number_of_bathrooms int,
sqft int,
location varchar(100),
days_on_market int,
initial_price int,
neighborhood varchar(100),
rental_price int
)
''')
with open(test_csv_path) as f:
csvf = csv.reader(f)
i = 0
for row in csvf:
if i > 0:
number_of_rooms = int(row[0])
number_of_bathrooms = int(row[1])
sqft = int(float(row[2].replace(',', '.')))
location = str(row[3])
days_on_market = int(row[4])
initial_price = int(row[5])
neighborhood = str(row[6])
rental_price = int(float(row[7]))
query(f'''INSERT INTO test.{TEST_DATA_TABLE} VALUES (
{number_of_rooms},
{number_of_bathrooms},
{sqft},
'{location}',
{days_on_market},
{initial_price},
'{neighborhood}',
{rental_price}
)''')
i += 1
print('done')
ds = datastore.get_datasource(EXTERNAL_DS_NAME)
if ds is not None:
datastore.delete_datasource(EXTERNAL_DS_NAME)
short_csv_file_path = get_test_csv(f'{EXTERNAL_DS_NAME}.csv', TEST_CSV['url'], lines_count=300, rewrite=True)
datastore.save_datasource(EXTERNAL_DS_NAME, 'file', 'test.csv', short_csv_file_path)
def test_2_broke_analisys(self):
'''
stop mindsdb while analyse dataset
'''
data = {
"integration_id": 'default_mariadb',
"name": TEST_DS,
"query": f"select * from test_data.{TEST_DATASET} limit 100;"
}
res = requests.put(f'{HTTP_API_ROOT}/datasources/{TEST_DS}', json=data)
assert res.status_code == 200
res = requests.get(f'{HTTP_API_ROOT}/datasources/{TEST_DS}/analyze')
assert res.status_code == 200
stop_mindsdb()
self.mdb, datastore = run_environment(
config,
apis=['mysql', 'http'],
override_integration_config={'default_mariadb': {
'publish': True
}},
mindsdb_database=MINDSDB_DATABASE,
clear_storage=False)
check_ds_exists(TEST_DS)
def setUpClass(cls):
run_environment(apis=['http'],
override_config={
'integrations': {
'default_mariadb': {
'publish': True
},
'default_clickhouse': {
'publish': True
}
}
})
cls.config = json.loads(Path(CONFIG_PATH).read_text())
cls.initial_integrations_names = list(
cls.config['integrations'].keys())
def setUpClass(cls):
mdb, datastore = run_environment(config,
apis=['mongodb'],
override_integration_config={
'default_mongodb': {
'publish': True,
'port': 27002,
'host': '127.0.0.1',
'type': 'mongodb',
'user': '',
'password': ''
}
},
mindsdb_database=MINDSDB_DATABASE)
cls.mdb = mdb
if USE_EXTERNAL_DB_SERVER:
open_ssh_tunnel(27002, direction='L') # 27002 - mongos port
wait_port(27002, timeout=10)
cls.mongos_client = MongoClient('mongodb://127.0.0.1:27002/')
mdb_shard = f"127.0.0.1:{config['api']['mongodb']['port']}"
try:
cls.mongos_client.admin.command('removeShard', mdb_shard)
except Exception:
# its ok if shard not exiss
pass
models = cls.mdb.get_models()
models = [x['name'] for x in models]
if TEST_PREDICTOR_NAME in models:
cls.mdb.delete_model(TEST_PREDICTOR_NAME)
if not USE_EXTERNAL_DB_SERVER:
test_csv_path = Path(DATASETS_PATH).joinpath(
TEST_DATASET).joinpath('data.csv')
db = cls.mongos_client['test_data']
colls = db.list_collection_names()
if TEST_DATASET not in colls:
print('creatating test data')
with open(test_csv_path) as f:
csvf = csv.reader(f)
data = []
DS = DATASETS_COLUMN_TYPES[TEST_DATASET]
for i, row in enumerate(csvf):
if i == 0:
continue
data.append({
column[0]: column[1](row[i])
for i, column in enumerate(DS)
})
db[TEST_DATASET].insert_many(data)
print('done')
cls.mongos_client.admin.command('addShard', mdb_shard)
def setUpClass(cls):
run_environment(apis=['mysql', 'http'],
override_config={
'api': {
'mysql': {
'ssl': False
}
},
'integrations': {
INTEGRATION_NAME: {
'publish': True
}
},
'permanent_storage': {
'location': 'local'
}
})
config.update(json.loads(Path(CONFIG_PATH).read_text()))
def test_1_wrong_integration(self):
'''
start mindsdb with publish integration with wrong password
try create ds
change password to correct
'''
original_db_password = config['integrations']['default_mariadb'][
'password']
self.mdb, datastore = run_environment(
config,
apis=['mysql', 'http'],
override_integration_config={
'default_mariadb': {
'publish': True,
'password': '******'
}
},
mindsdb_database=MINDSDB_DATABASE)
check_ds_not_exists(TEST_DS)
# TODO creating DS from unexists integration raise not critical error in code.
# need fix it and return human-readable error
# related issue: https://github.com/mindsdb/mindsdb/issues/945
# data = {
# "integration_id": 'unexists_integration',
# "name": TEST_DS,
# "query": f"select * from test_data.{TEST_DATASET} limit 50;"
# }
# res = requests.put(f'{HTTP_API_ROOT}/datasources/{TEST_DS}', json=data)
# assert res ?
# check create DS with wrong integration password
data = {
"integration_id": 'default_mariadb',
"name": TEST_DS,
"query": f"select * from test_data.{TEST_DATASET} limit 100;"
}
res = requests.put(f'{HTTP_API_ROOT}/datasources/{TEST_DS}', json=data)
assert 'Access denied for user' in res.json()['message']
check_ds_not_exists(TEST_DS)
# restore password
res = requests.post(
f'{HTTP_API_ROOT}/config/integrations/default_mariadb',
json={'params': {
'password': original_db_password
}})
assert res.status_code == 200
config['integrations']['default_mariadb'][
'password'] = original_db_password
def test_3_restart_and_connect(self):
stop_mindsdb()
mdb, datastore = run_environment(
config,
apis=['mysql'],
override_integration_config={
'default_mariadb': {
'publish': True
}
},
mindsdb_database=MINDSDB_DATABASE,
clear_storage=False
)
self.mdb = mdb
def setUpClass(cls):
mdb, datastore = run_environment(
config,
apis=['mysql'],
override_integration_config={
'default_clickhouse': {
'publish': True
}
},
override_api_config={'mysql': {
'ssl': False
}},
mindsdb_database=MINDSDB_DATABASE)
cls.mdb = mdb
models = cls.mdb.get_models()
models = [x['name'] for x in models]
if TEST_PREDICTOR_NAME in models:
cls.mdb.delete_model(TEST_PREDICTOR_NAME)
query('create database if not exists test_data')
if not USE_EXTERNAL_DB_SERVER:
test_csv_path = Path(DATASETS_PATH).joinpath(
TEST_DATASET).joinpath('data.csv')
if TEST_DATA_TABLE not in cls.get_tables_in(cls, 'test_data'):
print('creating test data table...')
upload_csv(
query=query,
columns_map=DATASETS_COLUMN_TYPES[TEST_DATASET],
db_types_map=DB_TYPES_MAP,
table_name=TEST_DATA_TABLE,
csv_path=test_csv_path,
template=
'create table test_data.%s (%s) ENGINE = MergeTree() ORDER BY days_on_market PARTITION BY location'
)
ds = datastore.get_datasource(EXTERNAL_DS_NAME)
if ds is not None:
datastore.delete_datasource(EXTERNAL_DS_NAME)
data = fetch(f'select * from test_data.{TEST_DATA_TABLE} limit 50')
external_datasource_csv = make_test_csv(EXTERNAL_DS_NAME, data)
datastore.save_datasource(EXTERNAL_DS_NAME, 'file', 'test.csv',
external_datasource_csv)
def setUpClass(cls):
cls.config = Config(TEST_CONFIG)
cls.initial_integrations_names = list(
cls.config['integrations'].keys())
mdb, datastore = run_environment(cls.config,
apis=['http'],
override_integration_config={
'default_mariadb': {
'publish': True
},
'default_clickhouse': {
'publish': True
}
},
mindsdb_database=MINDSDB_DATABASE)
cls.mdb = mdb
def test_1_wrong_integration(self):
'''
start mindsdb with publish integration with wrong password
try create ds
change password to correct
'''
original_db_password = config['integrations']['default_mariadb'][
'password']
self.mdb, datastore = run_environment(
config,
apis=['mysql', 'http'],
override_integration_config={
'default_mariadb': {
'publish': True,
'password': '******'
}
},
mindsdb_database=MINDSDB_DATABASE)
check_ds_not_exists(TEST_DS)
# check create DS with wrong integration password
data = {
"integration_id": 'default_mariadb',
"name": TEST_DS,
"query": f"select * from test_data.{TEST_DATASET} limit 100;"
}
res = requests.put(f'{HTTP_API_ROOT}/datasources/{TEST_DS}', json=data)
assert 'Access denied for user' in res.json()['message']
check_ds_not_exists(TEST_DS)
# restore password
res = requests.post(
f'{HTTP_API_ROOT}/config/integrations/default_mariadb',
json={'params': {
'password': original_db_password
}})
assert res.status_code == 200
config['integrations']['default_mariadb'][
'password'] = original_db_password
def setUpClass(cls):
mdb, datastore = run_environment(
config,
apis=['mysql'],
override_integration_config={'default_mssql': {
'publish': True
}},
mindsdb_database=MINDSDB_DATABASE)
cls.mdb = mdb
models = cls.mdb.get_models()
models = [x['name'] for x in models]
if TEST_PREDICTOR_NAME in models:
cls.mdb.delete_model(TEST_PREDICTOR_NAME)
data = fetch(
f'select * from test_data.{TEST_DATA_TABLE} order by rental_price offset 0 rows fetch next 50 rows only'
)
external_datasource_csv = make_test_csv(EXTERNAL_DS_NAME, data)
datastore.save_datasource(EXTERNAL_DS_NAME, 'file', 'test.csv',
external_datasource_csv)
def setUpClass(cls):
mdb, datastore = run_environment(config,
apis=['mysql', 'http'],
mindsdb_database=MINDSDB_DATABASE)
cls.mdb = mdb
query('create database if not exists test_data')
if not USE_EXTERNAL_DB_SERVER:
test_csv_path = Path(DATASETS_PATH).joinpath(
TEST_DATASET).joinpath('data.csv')
if TEST_DATA_TABLE not in cls.get_tables_in(cls, 'test_data'):
print('creating test data table...')
upload_csv(query=query,
columns_map=DATASETS_COLUMN_TYPES[TEST_DATASET],
db_types_map=DB_TYPES_MAP,
table_name=TEST_DATA_TABLE,
csv_path=test_csv_path)
data = fetch(f'select * from test_data.{TEST_DATA_TABLE} limit 50',
as_dict=True)
cls.external_datasource_csv_path = make_test_csv(
EXTERNAL_DS_NAME, data)
def setUpClass(cls):
mdb, datastore = run_environment(
config,
apis=['http', 'mysql'],
override_integration_config={'default_mariadb': {
'publish': True
}},
mindsdb_database=MINDSDB_DATABASE)
cls.mdb = mdb
models = cls.mdb.get_models()
models = [x['name'] for x in models]
if TEST_PREDICTOR_NAME in models:
cls.mdb.delete_model(TEST_PREDICTOR_NAME)
if not USE_EXTERNAL_DB_SERVER:
query('create database if not exists test_data')
test_tables = fetch('show tables from test_data', as_dict=False)
test_tables = [x[0] for x in test_tables]
if TEST_DATA_TABLE not in test_tables:
test_csv_path = Path(DATASETS_PATH).joinpath(
TEST_DATASET).joinpath('data.csv')
upload_csv(query=query,
columns_map=DATASETS_COLUMN_TYPES[TEST_DATASET],
db_types_map=DB_TYPES_MAP,
table_name=TEST_DATA_TABLE,
csv_path=test_csv_path)
ds = datastore.get_datasource(EXTERNAL_DS_NAME)
if ds is not None:
datastore.delete_datasource(EXTERNAL_DS_NAME)
data = fetch(f'select * from test_data.{TEST_DATA_TABLE} limit 50')
external_datasource_csv = make_test_csv(EXTERNAL_DS_NAME, data)
datastore.save_datasource(EXTERNAL_DS_NAME, 'file', 'test.csv',
external_datasource_csv)
def setUpClass(cls):
run_environment(apis=['http', ])
def setUpClass(cls):
mdb, datastore = run_environment(
config,
apis=['mysql'],
override_integration_config={'default_mssql': {
'publish': True
}},
mindsdb_database=MINDSDB_DATABASE)
cls.mdb = mdb
models = cls.mdb.get_models()
models = [x['name'] for x in models]
if TEST_PREDICTOR_NAME in models:
cls.mdb.delete_model(TEST_PREDICTOR_NAME)
test_csv_path = Path(DATASETS_PATH).joinpath('home_rentals').joinpath(
'data.csv')
res = query(
"SELECT name FROM master.dbo.sysdatabases where name = 'mindsdb_test'",
fetch=True)
if len(res) == 0:
query("create database mindsdb_test")
res = query('''
select * from sys.schemas where name = 'mindsdb_schema';
''',
fetch=True)
if len(res) == 0:
query('''
create schema [mindsdb_schema];
''')
if not USE_EXTERNAL_DB_SERVER:
# show tables from test
test_tables = query(f'''
select 1 from sysobjects where name='{TEST_DATA_TABLE}' and xtype='U';
''',
fetch=True)
if len(test_tables) == 0:
print('creating test data table...')
query(f'''
CREATE TABLE mindsdb_schema.{TEST_DATA_TABLE} (
number_of_rooms int,
number_of_bathrooms int,
sqft int,
location varchar(100),
days_on_market int,
initial_price int,
neighborhood varchar(100),
rental_price int
)
''')
with open(test_csv_path) as f:
csvf = csv.reader(f)
i = 0
for row in csvf:
if i > 0:
number_of_rooms = int(row[0])
number_of_bathrooms = int(row[1])
sqft = int(float(row[2].replace(',', '.')))
location = str(row[3])
days_on_market = int(row[4])
initial_price = int(row[5])
neighborhood = str(row[6])
rental_price = int(float(row[7]))
query(f'''
INSERT INTO mindsdb_schema.{TEST_DATA_TABLE} VALUES (
{number_of_rooms},
{number_of_bathrooms},
{sqft},
'{location}',
{days_on_market},
{initial_price},
'{neighborhood}',
{rental_price}
)''')
i += 1
if i % 100 == 0:
print(i)
print('done')
ds = datastore.get_datasource(EXTERNAL_DS_NAME)
if ds is not None:
datastore.delete_datasource(EXTERNAL_DS_NAME)
data = query(f'select * from test_data.{TEST_DATA_TABLE} limit 50',
fetch=True,
as_dict=True)
external_datasource_csv = make_test_csv(EXTERNAL_DS_NAME, data)
datastore.save_datasource(EXTERNAL_DS_NAME, 'file', 'test.csv',
external_datasource_csv)
def setUpClass(cls):
run_environment(apis=['mysql', 'http'])
def setUpClass(cls):
run_container('mongo-config')
ready = wait_port(27000, DOCKER_TIMEOUT)
assert ready
run_container('mongo-instance')
ready = wait_port(27001, DOCKER_TIMEOUT)
assert ready
cls.config_client = MongoClient('mongodb://localhost:27000/')
cls.instance_client = MongoClient('mongodb://localhost:27001/')
print('init replconf')
try:
r = cls.config_client.admin.command(
'replSetInitiate', {
'_id': 'replconf',
'members': [{
'_id': 0,
'host': '127.0.0.1:27000'
}]
})
except Exception as e:
print('already initialized')
if str(e) == 'already initialized':
r = {'ok': 1}
if bool(r['ok']) is not True:
assert False
print('init replmain')
try:
r = cls.instance_client.admin.command(
'replSetInitiate', {
'_id': 'replmain',
'members': [{
'_id': 0,
'host': '127.0.0.1:27001'
}]
})
except Exception as e:
print('already initialized')
if str(e) == 'already initialized':
r = {'ok': 1}
if bool(r['ok']) is not True:
assert False
mdb, datastore = run_environment('mongodb', config, run_apis='mongodb')
cls.mdb = mdb
models = cls.mdb.get_models()
models = [x['name'] for x in models]
if TEST_PREDICTOR_NAME in models:
cls.mdb.delete_model(TEST_PREDICTOR_NAME)
test_csv_path = get_test_csv(TEST_CSV['name'], TEST_CSV['url'])
db = cls.instance_client['test_data']
colls = db.list_collection_names()
if 'home_rentals' not in colls:
print('creatating test data')
with open(test_csv_path) as f:
csvf = csv.reader(f)
data = []
for i, row in enumerate(csvf):
if i > 0:
data.append(
dict(number_of_rooms=int(row[0]),
number_of_bathrooms=int(row[1]),
sqft=int(float(row[2].replace(',', '.'))),
location=str(row[3]),
days_on_market=int(row[4]),
initial_price=int(row[5]),
neighborhood=str(row[6]),
rental_price=int(float(row[7]))))
db['home_rentals'].insert_many(data)
print('done')
run_container('mongo-mongos')
ready = wait_port(27002, DOCKER_TIMEOUT)
assert ready
cls.mongos_client = MongoClient('mongodb://localhost:27002/')
cls.mongos_client.admin.command('addShard', 'replmain/127.0.0.1:27001')
cls.mongos_client.admin.command(
'addShard', f"127.0.0.1:{config['api']['mongodb']['port']}")
def setUpClass(cls):
mdb, datastore = run_environment('mssql', config)
cls.mdb = mdb
models = cls.mdb.get_models()
models = [x['name'] for x in models]
if TEST_PREDICTOR_NAME in models:
cls.mdb.delete_model(TEST_PREDICTOR_NAME)
test_csv_path = get_test_csv(TEST_CSV['name'], TEST_CSV['url'])
res = query(
"SELECT name FROM master.dbo.sysdatabases where name = 'mindsdb_test'",
fetch=True)
if len(res) == 0:
query("create database mindsdb_test")
res = query('''
select * from sys.schemas where name = 'mindsdb_schema';
''',
fetch=True)
if len(res) == 0:
query('''
create schema [mindsdb_schema];
''')
# query('create database if not exists test')
# show tables from test
test_tables = query(f'''
select 1 from sysobjects where name='{TEST_DATA_TABLE}' and xtype='U';
''',
fetch=True)
if len(test_tables) == 0:
print('creating test data table...')
query(f'''
CREATE TABLE mindsdb_schema.{TEST_DATA_TABLE} (
number_of_rooms int,
number_of_bathrooms int,
sqft int,
location varchar(100),
days_on_market int,
initial_price int,
neighborhood varchar(100),
rental_price int
)
''')
with open(test_csv_path) as f:
csvf = csv.reader(f)
i = 0
for row in csvf:
if i > 0:
number_of_rooms = int(row[0])
number_of_bathrooms = int(row[1])
sqft = int(float(row[2].replace(',', '.')))
location = str(row[3])
days_on_market = int(row[4])
initial_price = int(row[5])
neighborhood = str(row[6])
rental_price = int(float(row[7]))
query(f'''
INSERT INTO mindsdb_schema.{TEST_DATA_TABLE} VALUES (
{number_of_rooms},
{number_of_bathrooms},
{sqft},
'{location}',
{days_on_market},
{initial_price},
'{neighborhood}',
{rental_price}
)''')
i += 1
if i % 100 == 0:
print(i)
print('done')
ds = datastore.get_datasource(EXTERNAL_DS_NAME)
if ds is not None:
datastore.delete_datasource(EXTERNAL_DS_NAME)
short_csv_file_path = get_test_csv(f'{EXTERNAL_DS_NAME}.csv',
TEST_CSV['url'],
lines_count=300,
rewrite=True)
datastore.save_datasource(EXTERNAL_DS_NAME, 'file', 'test.csv',
short_csv_file_path)