def client():
config.ROOT_DIR = os.path.abspath('test_data')
config.DATASET_DIR = os.path.join(config.ROOT_DIR, 'dataset')
config.DATABASE_PATH = os.path.join(config.ROOT_DIR, 'database.db')
config.LOG_DIR = os.path.join(config.ROOT_DIR, 'train_logs')
database_path = config.DATABASE_PATH
app.config['TESTING'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///%s' % database_path
config.prepare_directory()
with app.app_context():
db.drop_all()
db.create_all()
return app.test_client()
def client():
config.ROOT_DIR = os.path.abspath("test_data")
config.DATASET_DIR = os.path.join(config.ROOT_DIR, "dataset")
config.DATABASE_PATH = os.path.join(config.ROOT_DIR, "database.db")
config.LOG_DIR = os.path.join(config.ROOT_DIR, "train_logs")
database_path = config.DATABASE_PATH
app.config["TESTING"] = True
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///%s" % database_path
config.prepare_directory()
with app.app_context():
db.drop_all()
db.create_all()
return app.test_client()
def test_project_api(client):
# upload dataset
res, _ = _upload_dataset(client)
dataset_id = res.json["id"]
# check create project
data = {"name": "test", "dataset_id": dataset_id, "algorithm": "cql"}
res = client.post(
"/api/projects",
data=json.dumps(data),
content_type="application/json",
follow_redirects=True,
)
assert res.status_code == 200
assert res.json["name"] == "test"
assert res.json["dataset_id"] == dataset_id
project_id = res.json["id"]
# check get
res = client.get("/api/projects/%d" % project_id, follow_redirects=True)
assert res.status_code == 200
assert res.json["id"] == project_id
assert res.json["name"] == "test"
assert res.json["dataset_id"] == dataset_id
# check get all
res = client.get("/api/projects", follow_redirects=True)
assert res.status_code == 200
assert len(res.json["projects"]) == 1
assert res.json["projects"][0]["id"] == project_id
# check update
res = client.put(
"/api/projects/%d" % project_id,
data=json.dumps({"name": "updated"}),
content_type="application/json",
follow_redirects=True,
)
assert res.status_code == 200
assert res.json["name"] == "updated"
# check delete
res = client.delete("/api/projects/%d" % project_id, follow_redirects=True)
assert res.status_code == 200
with app.app_context():
assert Project.get(project_id) is None
def test_project_api(client):
# upload dataset
res, _ = _upload_dataset(client)
dataset_id = res.json['id']
# check create project
data = {'name': 'test', 'dataset_id': dataset_id}
res = client.post('/api/projects',
data=json.dumps(data),
content_type='application/json',
follow_redirects=True)
assert res.status_code == 200
assert res.json['name'] == 'test'
assert res.json['dataset_id'] == dataset_id
project_id = res.json['id']
# check get
res = client.get('/api/projects/%d' % project_id, follow_redirects=True)
assert res.status_code == 200
assert res.json['id'] == project_id
assert res.json['name'] == 'test'
assert res.json['dataset_id'] == dataset_id
# check get all
res = client.get('/api/projects', follow_redirects=True)
assert res.status_code == 200
assert len(res.json['projects']) == 1
assert res.json['projects'][0]['id'] == project_id
# check update
res = client.put('/api/projects/%d' % project_id,
data=json.dumps({'name': 'updated'}),
content_type='application/json',
follow_redirects=True)
assert res.status_code == 200
assert res.json['name'] == 'updated'
# check delete
res = client.delete('/api/projects/%d' % project_id, follow_redirects=True)
assert res.status_code == 200
with app.app_context():
assert Project.get(project_id) is None
def test_experiment_api(client):
# upload dataset
res, _ = _upload_dataset(client)
# create project
data = {'name': 'test_project', 'dataset_id': res.json['id']}
res = client.post('/api/projects',
data=json.dumps(data),
content_type='application/json',
follow_redirects=True)
project_id = res.json['id']
# check create experiment
data = {'name': 'test_experiment', 'config': {'n_epochs': 1}}
res = client.post('/api/projects/%d/experiments' % project_id,
data=json.dumps(data),
content_type='application/json',
follow_redirects=True)
assert res.status_code == 200
assert res.json['name'] == 'test_experiment'
assert res.json['is_active']
experiment_id = res.json['id']
time.sleep(20)
# check get experiment
url = '/api/projects/%d/experiments/%d' % (project_id, experiment_id)
while True:
res = client.get(url, follow_redirects=True)
assert res.status_code == 200
if not res.json['is_active']:
break
time.sleep(1)
# check get all experiments
res = client.get('/api/projects/%d/experiments' % project_id)
assert res.status_code == 200
assert len(res.json['experiments']) == 1
# check update
url = '/api/projects/%d/experiments/%d' % (project_id, experiment_id)
res = client.put(url,
data=json.dumps({'name': 'updated'}),
content_type='application/json',
follow_redirects=True)
assert res.status_code == 200
assert res.json['name'] == 'updated'
# check download as TorchScript
url = '/api/projects/{}/experiments/{}/download?epoch=0&format=torchscript'
res = client.get(url.format(project_id, experiment_id))
assert res.status_code == 200
# check download as ONNX
url = '/api/projects/{}/experiments/{}/download?epoch=0&format=onnx'
res = client.get(url.format(project_id, experiment_id))
assert res.status_code == 200
# check delete
url = '/api/projects/%d/experiments/%d' % (project_id, experiment_id)
res = client.delete(url, follow_redirects=True)
assert res.status_code == 200
with app.app_context():
assert Experiment.get(experiment_id) is None
def test_dataset_api(client, is_image):
# check upload dataset
if is_image:
res, mdp_dataset = _upload_image_dataset(client)
else:
res, mdp_dataset = _upload_dataset(client)
assert res.status_code == 200
assert res.json['name'] == 'dataset.csv'
assert res.json['episode_size'] == len(mdp_dataset)
step_size = 0
for episode in mdp_dataset:
step_size += len(episode)
assert res.json['step_size'] == step_size
dataset_id = res.json['id']
dataset_name = res.json['name']
dataset_file_name = res.json['file_name']
dataset_path = os.path.join(config.DATASET_DIR, dataset_file_name)
assert os.path.exists(dataset_path)
# check get
res = client.get('/api/datasets/%d' % dataset_id, follow_redirects=True)
assert res.status_code == 200
assert res.json['id'] == dataset_id
assert res.json['name'] == dataset_name
assert res.json['file_name'] == dataset_file_name
# check get_all
res = client.get('/api/datasets', follow_redirects=True)
assert res.status_code == 200
assert len(res.json['datasets']) == 1
assert res.json['datasets'][0]['id'] == dataset_id
# check update
res = client.put('/api/datasets/%d' % dataset_id,
data=json.dumps({'name': 'updated'}),
content_type='application/json',
follow_redirects=True)
assert res.status_code == 200
assert res.json['name'] == 'updated'
with app.app_context():
assert Dataset.get(dataset_id).name == 'updated'
# check get example observations
res = client.get('/api/datasets/%d/example' % dataset_id)
assert res.status_code == 200
examples = res.json['observations']
if is_image:
for i, base64_image in enumerate(examples):
image = Image.open(BytesIO(base64.b64decode(base64_image)))
ndarray = convert_image_to_ndarray(image)
assert np.all(ndarray == mdp_dataset.observations[i])
else:
assert np.allclose(np.array(examples), mdp_dataset.observations[:100])
# check delete
res = client.delete('/api/datasets/%d' % dataset_id, follow_redirects=True)
assert res.status_code == 200
with app.app_context():
assert Dataset.get(dataset_id) is None
assert not os.path.exists(dataset_path)
def test_experiment_api(client):
# upload dataset
res, _ = _upload_dataset(client)
# create project
data = {
"name": "test_project",
"dataset_id": res.json["id"],
"algorithm": "cql",
}
res = client.post(
"/api/projects",
data=json.dumps(data),
content_type="application/json",
follow_redirects=True,
)
project_id = res.json["id"]
# check create experiment
experiment_config = {"n_epochs": 1, "n_steps_per_epoch": 10}
data = {"name": "test_experiment", "config": experiment_config}
res = client.post(
"/api/projects/%d/experiments" % project_id,
data=json.dumps(data),
content_type="application/json",
follow_redirects=True,
)
assert res.status_code == 200
assert res.json["name"] == "test_experiment"
assert res.json["is_active"]
experiment_id = res.json["id"]
time.sleep(20)
# check get experiment
url = "/api/projects/%d/experiments/%d" % (project_id, experiment_id)
while True:
res = client.get(url, follow_redirects=True)
assert res.status_code == 200
if not res.json["is_active"]:
break
time.sleep(1)
# check get all experiments
res = client.get("/api/projects/%d/experiments" % project_id)
assert res.status_code == 200
assert len(res.json["experiments"]) == 1
# check update
url = "/api/projects/%d/experiments/%d" % (project_id, experiment_id)
res = client.put(
url,
data=json.dumps({"name": "updated"}),
content_type="application/json",
follow_redirects=True,
)
assert res.status_code == 200
assert res.json["name"] == "updated"
# check download as TorchScript
url = "/api/projects/{}/experiments/{}/download?epoch=1&format=torchscript"
res = client.get(url.format(project_id, experiment_id))
assert res.status_code == 200
# check download as ONNX
url = "/api/projects/{}/experiments/{}/download?epoch=1&format=onnx"
res = client.get(url.format(project_id, experiment_id))
assert res.status_code == 200
# check delete
url = "/api/projects/%d/experiments/%d" % (project_id, experiment_id)
res = client.delete(url, follow_redirects=True)
assert res.status_code == 200
with app.app_context():
assert Experiment.get(experiment_id) is None
def test_dataset_api(client, is_image):
# check upload dataset
if is_image:
res, mdp_dataset = _upload_image_dataset(client)
else:
res, mdp_dataset = _upload_dataset(client)
assert res.status_code == 200
assert res.json["name"] == "dataset.csv"
assert res.json["episode_size"] == len(mdp_dataset)
step_size = 0
for episode in mdp_dataset:
step_size += len(episode)
assert res.json["step_size"] == step_size
dataset_id = res.json["id"]
dataset_name = res.json["name"]
dataset_file_name = res.json["file_name"]
dataset_path = os.path.join(config.DATASET_DIR, dataset_file_name)
assert os.path.exists(dataset_path)
# check get
res = client.get("/api/datasets/%d" % dataset_id, follow_redirects=True)
assert res.status_code == 200
assert res.json["id"] == dataset_id
assert res.json["name"] == dataset_name
assert res.json["file_name"] == dataset_file_name
assert res.json["is_discrete"] != is_image
# check get_all
res = client.get("/api/datasets", follow_redirects=True)
assert res.status_code == 200
assert len(res.json["datasets"]) == 1
assert res.json["datasets"][0]["id"] == dataset_id
# check update
res = client.put(
"/api/datasets/%d" % dataset_id,
data=json.dumps({"name": "updated"}),
content_type="application/json",
follow_redirects=True,
)
assert res.status_code == 200
assert res.json["name"] == "updated"
with app.app_context():
assert Dataset.get(dataset_id).name == "updated"
# check get example observations
res = client.get("/api/datasets/%d/example" % dataset_id)
assert res.status_code == 200
examples = res.json["observations"]
if is_image:
for i, base64_image in enumerate(examples):
image = Image.open(BytesIO(base64.b64decode(base64_image)))
ndarray = convert_image_to_ndarray(image)
assert np.all(ndarray == mdp_dataset.observations[i])
else:
assert np.allclose(np.array(examples), mdp_dataset.observations[:100])
# check delete
res = client.delete("/api/datasets/%d" % dataset_id, follow_redirects=True)
assert res.status_code == 200
with app.app_context():
assert Dataset.get(dataset_id) is None
assert not os.path.exists(dataset_path)
