def create_project(self):
organization = Organization.objects.get(
slug=self.user1_params["organization_slug"]
)
user = MljarUser.objects.get(email=self.user1_params["email"])
project = Project(
title="some title",
description="...",
created_by=user,
parent_organization=organization,
)
project.save()
return project
def test_project_created(self, project_created):
"""Ensure that the `project_created` task is run with the correct arguments when a new Project object is created
and saved.
"""
project = Project(name='Test Project',
authors=['Author 1'],
description='Project description.',
status=Project.ProjectStatus.PLANNED)
project.save()
self.assertTrue(project_created.called)
self.assertEqual(project.name, project_created.call_args[0][0])
self.assertEqual(project.authors, project_created.call_args[0][1])
self.assertEqual(project.description, project_created.call_args[0][2])
self.assertEqual(project.url, project_created.call_args[0][3])
def test_image_file_deleted_with_object(self):
"""Ensure that a project's image is deleted from the disk when its associated Project object is deleted.
"""
project = Project(
name='Test Project',
authors=['Author 1', 'Author 2'],
description='Project description.',
image=self.image,
status=Project.ProjectStatus.PLANNED
)
project.save()
path = f'{project.image.path}'
project.delete()
import os
if os.path.isfile(path):
self.fail('Project image file was not deleted along with the model instance.')
def test_logo_path_properly_assigned(self):
"""Ensure that the URL and name of a project's image are properly assigned when creating a Project object.
"""
project = Project(
name='Test Project',
authors=['Author 1', 'Author 2'],
description='Project description.',
image=self.image,
status=Project.ProjectStatus.PLANNED
)
project.save()
try:
self.assertEqual(f'{settings.MEDIA_URL}{BASE_IMAGE_PATH}test_project/main_image.png', project.image.url)
except AssertionError as e:
project.delete()
self.fail(e)
project.delete()
def test_change_status_to_finished(self):
project = Project(start_date='2011-01-01', end_date=None)
project |should_not| be_finished
project.end_date = '2011-01-31'
project.save()
project.status |should| equal_to('finalizado')
def test_start_mlexperiment(self):
token = self.create_user_and_login(self.user1_params)
organization = Organization.objects.get(slug=self.org1)
user = MljarUser.objects.get(email=self.user1_params["email"])
project = Project(
title="some title",
description="...",
created_by=user,
parent_organization=organization,
)
project.save()
# prepare data
local_full_file_path = "/tmp/example.csv"
filename = "example.csv"
relative_dir = "test"
absolute_path = Storage().get_path(relative_dir, filename)
data_to_file(example_X, example_y, absolute_path)
ds = FileDataSource(
title="my file",
description="desc ...",
absolute_path=absolute_path,
file_name=filename,
file_size=os.path.getsize(absolute_path),
created_by=user,
parent_organization=organization,
parent_project=project,
)
ds.save()
job_params = {
"absolute_path": ds.absolute_path,
"file_name": ds.file_name,
"db_id": ds.id,
"created_by_id": ds.created_by.id,
"parent_organization_id": ds.parent_organization.id,
"parent_project_id": ds.parent_project.id,
}
self.assertEqual(DataFrame.objects.all().count(), 0)
process_file = ProcessUploadedFile(job_params)
process_file.run()
time.sleep(1) # not nice but till no websockets, let's use it
self.assertEqual(DataFrame.objects.all().count(), 1)
self.assertTrue(DataFrame.objects.filter(source_id=ds.id).count(), 1)
mljar_df = DataFrame.objects.get(source_id=ds.id)
print(mljar_df.columns_details)
### start ml experiment ###
mlexperiment = MLExperiment(
title="exp 1",
description="na na ...",
params={
"data_usage": {
"train_absolute_path": mljar_df.absolute_path
},
"metric": {
"optimize": "logloss",
"monitor": ["logloss", "auc"]
},
"validation": {
"validation_type": "split",
"train_ratio": 0.5,
"shuffle": True,
},
"preprocessing": {},
},
column_usage={
"target": ["target"],
"input": ["feature_{0}".format(i) for i in range(4)],
},
created_by_id=user.id,
parent_organization_id=organization.id,
parent_project_id=project.id,
)
mlexperiment.save()
job_params = {
"db_id": mlexperiment.id,
"params": mlexperiment.params,
"column_usage": mlexperiment.column_usage,
"created_by_id": mlexperiment.created_by.id,
"parent_organization_id": mlexperiment.parent_organization.id,
"parent_project_id": mlexperiment.parent_project.id,
}
automl = StartMLExperiment(job_params)
automl.run()
mlexperiment = MLExperiment.objects.get(pk=mlexperiment.id)
print(mlexperiment.status)
time.sleep(1.5)
mlexperiment = MLExperiment.objects.get(pk=mlexperiment.id)
print(mlexperiment.status)
self.assertEqual(mlexperiment.status, "done")
def test_change_status_to_finished(self):
project = Project(start_date='2011-01-01', end_date=None)
project | should_not | be_finished
project.end_date = '2011-01-31'
project.save()
project.status | should | equal_to('finalizado')
def test_preprocess(self):
# set user
token = self.create_user_and_login(self.user1_params)
organization = Organization.objects.get(slug=self.org1)
user = MljarUser.objects.get(email=self.user1_params["email"])
project = Project(
title="some title",
description="...",
created_by=user,
parent_organization=organization,
)
project.save()
# prepare data
local_full_file_path = "/tmp/example.csv"
filename = "example.csv"
relative_dir = "test"
absolute_path = Storage().get_path(relative_dir, filename)
data_to_file(example_X, example_y, absolute_path)
ds = FileDataSource(
title="my file",
description="desc ...",
absolute_path=absolute_path,
file_name=filename,
file_size=os.path.getsize(absolute_path),
created_by=user,
parent_organization=organization,
parent_project=project,
)
ds.save()
job_params = {
"absolute_path": ds.absolute_path,
"file_name": ds.file_name,
"db_id": ds.id,
"created_by_id": ds.created_by.id,
"parent_organization_id": ds.parent_organization.id,
"parent_project_id": ds.parent_project.id,
}
self.assertEqual(DataFrame.objects.all().count(), 0)
########################################################################
# run job
########################################################################
process_file = ProcessUploadedFile(job_params)
process_file.run()
########################################################################
# check if all is good
self.assertEqual(DataFrame.objects.all().count(), 1)
self.assertTrue(DataFrame.objects.filter(source_id=ds.id).count(), 1)
dataframe = DataFrame.objects.get(source_id=ds.id)
preview = self.request(
method="get",
endpoint="/api/v1/{0}/{1}/dataframe_preview/{2}".format(
self.org1, project.id, dataframe.id),
payload={},
token=token,
expected_status_code=200,
)
self.assertEqual(len(json.loads(preview.get("preview_data"))), 100)
self.assertTrue("columns_description" in preview)
self.assertTrue("nrows" in preview)
self.assertTrue("ncols" in preview)
frames = self.request(
method="get",
endpoint="/api/v1/{0}/{1}/dataframes".format(
self.org1, project.id),
payload={},
token=token,
expected_status_code=200,
)
print(frames)
