def post(self):
import requests
job_ids = self.params.get('job_ids', None)
if job_ids is None:
return Response(
'Bad Request',
LazyResult(
lambda: 'Expected a json object with key \'job_ids\''),
status=400)
tb_locations = self._transform_request(job_ids)
response = requests.post(
f'{self._tensorboard_api_host()}/create_sym_links',
json=tb_locations)
if response.status_code == 400:
return Response('Bad Request',
LazyResult(lambda: response.text),
status=400)
elif response.status_code == 500:
return Response('Tensorboard Rest API Error',
LazyResult(lambda: 'Internal Server Error'),
status=500)
return Response(
'Jobs', LazyResult(lambda: {'url': f'{self._tensorboard_host()}'}))
def delete(self):
from foundations_core_rest_api_components.response import Response
from foundations_core_rest_api_components.lazy_result import LazyResult
from foundations_contrib.global_state import config_manager
job_deployment_class = config_manager['deployment_implementation'][
'deployment_type']
job_deployment = job_deployment_class(self.job_id, None, None)
job_status = job_deployment.get_job_status()
if job_status is None:
return Response(
'Jobs',
LazyResult(
lambda: f'Job {self.job_id} is in an unknown state'))
if not (job_status == 'completed' or job_status == 'failed'):
return Response(
'Jobs',
LazyResult(
lambda: f'Job {self.job_id} is not completed or failed'))
if all(job_deployment.cancel_jobs([self.job_id]).values()):
return Response(
'Jobs',
LazyResult(lambda: f'Job {self.job_id} successfully deleted'))
return Response(
'Jobs',
LazyResult(lambda: f'Job {self.job_id} could not be deleted'))
def test_recursive_lazy_result_via_list(self):
mock = self.Mock('hello world')
lazy_result = LazyResult(mock.value)
mock2 = self.Mock([lazy_result])
lazy_result2 = LazyResult(mock2.value)
self.assertEqual(['hello world'], lazy_result2.evaluate())
def test_recursive_lazy_result_via_dictionary(self):
mock = self.Mock('hello world')
lazy_result = LazyResult(mock.value)
mock2 = self.Mock({'data': lazy_result})
lazy_result2 = LazyResult(mock2.value)
self.assertEqual({'data': 'hello world'}, lazy_result2.evaluate())
def test_recursive_lazy_result(self):
mock = self.Mock('hello world')
lazy_result = LazyResult(mock.value)
mock2 = self.Mock(lazy_result)
lazy_result2 = LazyResult(mock2.value)
self.assertEqual('hello world', lazy_result2.evaluate())
def test_lazy_result_only_does_not_evaluate_other_properties(self):
should_not_call_mock = self.ErrorMock()
should_not_call_result = LazyResult(should_not_call_mock.value)
mock = self.Mock(
self.MockModelTwo(some_data='hello',
some_other_data=should_not_call_result))
lazy_result = LazyResult(mock.value)
lazy_result.only(['some_data']).evaluate()
def index(self):
from foundations_contrib.jobs.logs import job_logs
try:
logs = job_logs(self.params['job_id'])
return Response('Jobs', LazyResult(lambda: {'log': logs}))
except Exception as exc:
return Response(
'Error',
LazyResult(lambda: {'message': 'Internal Server Error'}),
status=500)
def test_lazy_result_only_filter_properties_in_list_of_lazy_results(self):
mock = self.Mock(
self.MockModelTwo(some_data='hello', some_other_data='world'))
lazy_result = LazyResult(mock.value)
mock_two = self.Mock([lazy_result])
lazy_result = LazyResult(mock_two.value)
self.assertEqual([{
'some_data': 'hello'
}],
lazy_result.only(['some_data']).evaluate())
def test_lazy_result_only_does_not_filter_too_much(self):
parent_mock = self.Mock(self.MockModel(data='hello'))
parent_lazy_result = LazyResult(parent_mock.value)
mock = self.Mock(
self.MockModelTwo(some_data=parent_lazy_result,
some_other_data='world'))
lazy_result = LazyResult(mock.value)
result = lazy_result.only(['some_data']).evaluate()
self.assertEqual(['some_data'], list(result.keys()))
self.assertEqual({'data': 'hello'}, result['some_data'].attributes)
def test_recursive_lazy_result_via_lazy_result_in_property_containing_model(
self):
mock = self.Mock([self.MockModel(data='hello world')])
lazy_result = LazyResult(mock.value)
mock2 = self.Mock(self.MockModel(data=lazy_result))
lazy_result2 = LazyResult(mock2.value)
result_attributes = lazy_result2.evaluate().attributes
self.assertEqual(['data'], list(result_attributes.keys()))
self.assertEqual({'data': 'hello world'},
result_attributes['data'][0].attributes)
def test_all_returns_correct_output_metric_names(self, mock_projects,
mock_jobs):
from foundations_core_rest_api_components.lazy_result import LazyResult
mock_jobs.return_value = LazyResult(lambda: [
self.MockJob(
job_id="123",
job_parameters=[],
output_metrics=[{
"name": "metric_1",
"type": "list"
}],
)
])
mock_projects.list_projects.return_value = [{"name": "project1"}]
project = Project.all().evaluate()[0]
expected_project = Project(name="project1",
created_at=None,
owner=None,
jobs=[
self.MockJob(job_id="123",
job_parameters=[],
output_metrics=[])
],
output_metric_names=[{
"name": "metric_1",
"type": "list"
}],
parameters=[])
self.assertCountEqual(expected_project.output_metric_names,
project.output_metric_names)
def all(job_id=None):
from foundations_core_rest_api_components.lazy_result import LazyResult
def _all():
return list(JobArtifact._all_artifacts(job_id))
return LazyResult(_all)
def all(project_name=None):
from foundations_core_rest_api_components.lazy_result import LazyResult
def _all():
return Job._all_internal(project_name)
return LazyResult(_all)
def _make_lazy_result(self, name):
from foundations_core_rest_api_components.lazy_result import LazyResult
from foundations_core_rest_api_components.v1.models.project import Project
def _callback():
return [Project(name=name)]
return LazyResult(_callback)
def all():
from foundations_core_rest_api_components.lazy_result import LazyResult
def callback():
listing = Project._construct_project_listing()
return [Project.find_by(project) for project in listing]
return LazyResult(callback)
def test_map_evaluates_lazy_result_and_callback(self):
mock = self.Mock('hello world')
lazy_result = LazyResult(mock.value)
def _callback(result):
return result + ' result'
self.assertEqual(
lazy_result.map(_callback).evaluate(), 'hello world result')
def _mock_resource(self, method, callback, status=200, cookie=None):
from foundations_core_rest_api_components.lazy_result import LazyResult
mock_klass = Mock()
mock_instance = Mock()
mock_klass.return_value = mock_instance
result = LazyResult(lambda: callback(mock_instance))
getattr(mock_instance, method).side_effect = lambda: Response('Mock', result, status=status, cookie=cookie)
return mock_klass
def test_lazy_result_only_filter_properties_multiple_properties(self):
mock = self.Mock(
self.MockModelTwo(some_data='hello', some_other_data='world'))
lazy_result = LazyResult(mock.value)
self.assertEqual({
'some_data': 'hello',
'some_other_data': 'world'
},
lazy_result.only(['some_data',
'some_other_data']).evaluate())
def delete(self):
from foundations_rest_api.global_state import redis_connection
job_annotations_key = f"jobs:{self._job_id()}:annotations"
redis_connection.hdel(job_annotations_key, self._key())
return Response(
"Jobs",
LazyResult(
lambda:
f"Tag key: {self._key()} deleted from job {self._job_id()}"),
)
def test_all_filters_out_non_hyperparameters_and_does_not_append_suffix_if_flag_false(
self, mock_get_all_jobs_data):
from test.datetime_faker import fake_current_datetime, restore_real_current_datetime
fake_current_datetime(1005000000)
self.mock_get_all_artifacts.return_when(LazyResult(lambda: []),
job_id='my job x')
mock_get_all_jobs_data.return_value = [
{
'project_name': 'random test project',
'job_id': 'my job x',
'user': '******',
'job_parameters': {
'hello': 'world'
},
'output_metrics': [],
'status': 'completed',
'start_time': 123456789,
'completed_time': 2222222222,
'tags': {
'beep': 'boop'
},
'artifacts': []
},
]
expected_job_1 = Job(job_id='my job x',
project='random test project',
user='******',
output_metrics=[],
job_parameters=[{
'name': 'hello',
'value': 'world',
'type': 'string'
}],
status='completed',
start_time='1973-11-29T21:33:09',
completed_time='2040-06-02T03:57:02',
duration='24291d6h23m53s',
tags={'beep': 'boop'},
artifacts=[])
result = Job.all(project_name='random test project').evaluate()
restore_real_current_datetime()
expected_jobs = [expected_job_1]
self.assertEqual(expected_jobs, result)
def index(self):
from foundations_rest_api.v2beta.models.project import Project
from foundations_core_rest_api_components.response import Response
from foundations_core_rest_api_components.lazy_result import LazyResult
project_name = self.params.pop("project_name")
jobs_data_future = Project.find_by(name=project_name).only(
["name", "jobs", "output_metric_names", "parameters"])
jobs_data_future = jobs_data_future.apply_filters(self.params,
fields=["jobs"])
fallback = Response("Jobs",
LazyResult(lambda: "This project was not found"),
status=404)
return Response("Jobs", jobs_data_future, fallback=fallback)
def _make_lazy_result(self,
name,
jobs,
output_metric_names=None,
parameters=None):
from foundations_core_rest_api_components.lazy_result import LazyResult
def _callback():
return self.MockProject(name=name,
jobs=jobs,
output_metric_names=output_metric_names
or [],
parameters=parameters or [])
return LazyResult(_callback)
def find_by(name):
"""Finds a project by name
Arguments:
name {str} -- Name of the project to find
Returns:
Project -- The project
"""
from foundations_core_rest_api_components.lazy_result import LazyResult
def callback():
return Project._find_by_internal(name)
return LazyResult(callback)
def new(name):
"""Creates a new instance of a Project given a set of properties
Arguments:
name {str} -- Name of the project
Returns:
Project -- The new instance of the project
"""
from foundations_core_rest_api_components.lazy_result import LazyResult
def callback():
return Project(name=name)
return LazyResult(callback)
def index(self):
from foundations_rest_api.v2beta.models.project import Project
from foundations_core_rest_api_components.response import Response
from foundations_core_rest_api_components.lazy_result import LazyResult
project_name = self.params.pop('project_name')
jobs_data_future = Project.find_by(name=project_name).only(
['name', 'jobs', 'output_metric_names'])
jobs_data_future = jobs_data_future.apply_filters({}, fields=['jobs'])
fallback = Response(
'Jobs',
LazyResult(lambda: 'This project or sort detail was not found'),
status=404)
jobs_data_future = self.sort_jobs(jobs_data_future)
return Response('Jobs', jobs_data_future, fallback=fallback)
def test_all_returns_all_projects_multiple_projects(
self, mock_projects, mock_jobs):
from foundations_core_rest_api_components.lazy_result import LazyResult
mock_jobs.return_value = LazyResult(
lambda:
[self.MockJob(job_id="123", job_parameters=[], output_metrics=[])])
mock_projects.list_projects.return_value = [
{
"name": "project1"
},
{
"name": "project2"
},
]
projects = [project for project in Project.all().evaluate()]
expected_project = Project(name="project1",
created_at=None,
owner=None,
jobs=[
self.MockJob(job_id="123",
job_parameters=[],
output_metrics=[])
],
output_metric_names=[],
parameters=[])
expected_project_two = Project(name="project2",
created_at=None,
owner=None,
jobs=[
self.MockJob(job_id="123",
job_parameters=[],
output_metrics=[])
],
output_metric_names=[],
parameters=[])
self.assertEqual([expected_project, expected_project_two], projects)
def post(self):
from foundations_rest_api.global_state import redis_connection
from foundations_internal.fast_serializer import serialize
from datetime import datetime
redis_connection.rpush(
f"project:{self._project_name()}:note_listing",
serialize(
{
"date": datetime.now(),
"message": self._message(),
"author": self._author(),
}
),
)
return Response(
"Note Listing",
LazyResult(
lambda: f"Note with author: {self._author()} created with message: {self._message()}"
),
)
def test_all_returns_multiple_jobs_with_artifacts(self, mock_get_all_jobs_data):
from test.datetime_faker import fake_current_datetime, restore_real_current_datetime
fake_current_datetime(1005000000)
self.mock_get_all_artifacts.return_when(LazyResult(lambda: [
JobArtifact(filename='output_x.png',uri='output_x.png', artifact_type='png')
]), job_id='my job x')
self.mock_get_all_artifacts.return_when(LazyResult(lambda: [
JobArtifact(filename='output_v007.wav', uri='output_v007.wav', artifact_type='wav'),
JobArtifact(filename='output_v007.txt', uri='output_v007.txt', artifact_type='unknown'),
]), job_id='00000000-0000-0000-0000-000000000007')
mock_get_all_jobs_data.return_value = [
{
'project_name': 'random test project',
'job_id': 'my job x',
'user': '******',
'job_parameters': {},
'output_metrics': [],
'status': 'completed',
'start_time': 123456789,
'completed_time': 2222222222,
'tags': {},
'artifacts': [{'filename': 'output_x.png', 'uri': 'output_x.png', 'artifact_type': 'png'}]
},
{
'project_name': 'random test project',
'job_id': '00000000-0000-0000-0000-000000000007',
'user': '******',
'job_parameters': {},
'output_metrics': [],
'status': 'running',
'start_time': 999999999,
'completed_time': None,
'tags': {
'asdf': 'this',
'cool': 'dude'
},
'artifacts': [
{'filename': 'output_v007.wav', 'uri': 'output_v007.wav', 'artifact_type': 'wav'},
{'filename': 'output_v007.txt', 'uri': 'output_v007.txt', 'artifact_type': 'unknown'},
]
}
]
expected_job_1 = Job(
job_id='00000000-0000-0000-0000-000000000007',
project='random test project',
user='******',
output_metrics=[],
job_parameters=[],
status='running',
start_time='2001-09-09T01:46:39',
completed_time=None,
duration='58d1h53m21s',
tags={
'asdf': 'this',
'cool': 'dude'
},
artifacts=[
JobArtifact(filename='output_v007.wav', uri='output_v007.wav', artifact_type='wav'),
JobArtifact(filename='output_v007.txt', uri='output_v007.txt', artifact_type='unknown')
]
)
expected_job_2 = Job(
job_id='my job x',
project='random test project',
user='******',
output_metrics=[],
job_parameters=[],
status='completed',
start_time='1973-11-29T21:33:09',
completed_time='2040-06-02T03:57:02',
duration='24291d6h23m53s',
tags={},
artifacts=[JobArtifact(filename='output_x.png',uri='output_x.png', artifact_type='png')]
)
result = Job.all(project_name='random test project').evaluate()
restore_real_current_datetime()
expected_jobs = [expected_job_1, expected_job_2]
self.assertEqual(expected_jobs, result)
def _constant_lazy_result(value):
from foundations_core_rest_api_components.lazy_result import LazyResult
return LazyResult(lambda: value)
def delete(self):
from foundations_core_rest_api_components.lazy_result import LazyResult
from foundations_core_rest_api_components.response import Response
def _delete():
return self.params
return Response('Mock', LazyResult(_delete))