class Manifestation(Resource):
data = JSONField(null=True)
community = GenericForeignKey(ct_field="community_type",
fk_field="community_id")
community_type = models.ForeignKey(ContentType, related_name="+")
community_id = models.PositiveIntegerField()
task = models.CharField(max_length=255, null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
@staticmethod
def generate_config(allowed_config, **kwargs):
warnings.warn(
"Manifestation.generate_config deprecated in favor for "
"Community.filter_growth_configuration and Community.filter_scope_configuration"
)
config = {
key: value
for key, value in kwargs.items() if key in allowed_config
}
return config
def get_data(self, async=False):
from core.tasks import get_manifestation_data
if self.data:
return self.data
if self.task:
result = AsyncResult(self.task)
if not result.ready():
raise DSProcessUnfinished("Manifest processing is not done")
elif result.successful():
self.data = result.result
elif result.failed():
self.data = str(result.result)
else:
raise AssertionError(
"get_data is not handling AsyncResult with status: {}".
format(result.status))
self.status = celery_status_code(result.status)
elif async:
self.task = get_manifestation_data.delay(self.id)
self.status = celery_status_code(PENDING)
self.save()
raise DSProcessUnfinished("Manifest started processing")
else:
self.data = get_manifestation_data(self.id)
self.status = celery_status_code(SUCCESS)
self.completed_at = datetime.now()
self.save()
return self.data
def async_results(result_id):
async_result = AsyncResult(result_id)
if not async_result.ready():
raise DSProcessUnfinished("Result with id {} is not ready.".format(result_id))
if async_result.status != TaskStates.SUCCESS:
raise DSProcessError("An error occurred during background processing.")
return async_result.result
def get_response(self, community_class, query_path, query_parameters):
assert isinstance(query_parameters, dict), \
"query_parameters for get_response should be a dictionary without urlencoded values"
response_data = copy(self.RESPONSE_DATA)
full_path = self.get_full_path(community_class, query_path,
query_parameters)
try:
manifestation = Manifestation.objects.get(uri=full_path)
community = manifestation.community
except Manifestation.DoesNotExist:
manifestation = None
signature = community_class.get_signature_from_input(
*query_path.split('/'), **query_parameters)
created_at = parse_datetime_string(query_parameters.get("t", None))
if "t" not in query_parameters:
community, created = community_class.objects.get_latest_or_create_by_signature(
signature, **query_parameters)
elif created_at is not None:
community = community_class.objects.get(signature=signature,
created_at=created_at)
community.config = community_class.get_configuration_from_input(
**query_parameters)
else:
raise Http404("Can not find community with t={}".format(
query_parameters.get("t")))
try:
if manifestation is not None:
return self._get_response_from_manifestation(
manifestation, response_data)
if community.state == CommunityState.SYNC:
raise DSProcessUnfinished()
community.grow(*query_path.split('/'))
config = Manifestation.generate_config(community.PUBLIC_CONFIG,
**query_parameters)
manifestation = Manifestation.objects.create(uri=full_path,
community=community,
config=config)
return self._get_response_from_manifestation(
manifestation, response_data)
except ValidationError as exc:
response_data["error"] = exc
return Response(response_data, HTTP_400_BAD_REQUEST)
except DSProcessUnfinished:
# FEATURE: set status
return Response(response_data, HTTP_202_ACCEPTED)
except DSProcessError:
# FEATURE: set errors
return Response(response_data, HTTP_500_INTERNAL_SERVER_ERROR)
def get_response(self, community_class, query_path, configuration, created_at_info=(None, None)):
assert isinstance(configuration, dict), \
"configuration for get_response should be a dictionary without urlencoded values"
created_at_parameter, created_at = created_at_info
uri = CommunityView.get_uri(community_class, query_path, configuration, created_at)
try:
manifestation = Manifestation.objects.get(uri=uri)
community = manifestation.community
except Manifestation.DoesNotExist:
manifestation = None
signature = community_class.get_signature_from_input(
*query_path.split('/'),
**configuration
)
if created_at is None:
community, created = community_class.objects.get_latest_or_create_by_signature(
signature,
**configuration
)
else:
try:
community = community_class.objects.get(signature=signature, created_at=created_at)
community.config = community_class.filter_growth_configuration(**configuration)
except community_class.DoesNotExist:
raise Http404("Can not find community with t={}".format(created_at_parameter))
try:
if manifestation is not None:
return self._get_response_from_manifestation(manifestation)
if community.state == CommunityState.SYNC:
raise DSProcessUnfinished()
community.grow(*query_path.split('/'))
config = community.filter_scope_configuration(**configuration)
manifestation = Manifestation.objects.create(uri=uri, community=community, config=config)
return self._get_response_from_manifestation(manifestation)
except ValidationError as exc:
return self._get_response_from_error(exc.message, HTTP_400_BAD_REQUEST)
except DSProcessUnfinished:
# FEATURE: set status
response_data = copy(self.RESPONSE_DATA)
return Response(response_data, HTTP_202_ACCEPTED)
except DSProcessError as exc:
# TODO: log exception
return self._get_response_from_error(str(exc), HTTP_500_INTERNAL_SERVER_ERROR)
class Manifestation(Resource):
data = JSONField(null=True)
community = GenericForeignKey(ct_field="community_type",
fk_field="community_id")
community_type = models.ForeignKey(ContentType, related_name="+")
community_id = models.PositiveIntegerField()
task = models.CharField(max_length=255, null=True, blank=True)
completed_at = models.DateTimeField(null=True, blank=True)
@staticmethod
def generate_config(allowed_config, **kwargs):
config = {
key: value
for key, value in kwargs.items() if key in allowed_config
}
return config
def get_data(self, async=False):
from core.tasks import get_manifestation_data
if self.data:
return self.data
if self.task:
result = AsyncResult(self.task)
if not result.ready():
raise DSProcessUnfinished("Manifest processing is not done")
self.data = result.result
elif async:
self.task = get_manifestation_data.delay(self.id)
self.save()
raise DSProcessUnfinished("Manifest started processing")
else:
self.data = get_manifestation_data(self.id)
self.completed_at = datetime.now()
self.save()
return self.data
class TestHtmlCommunityView(TestCase):
fixtures = ["test-community"]
def setUp(self):
super().setUp()
self.client = Client()
self.ready_url = "/data/v1/mock/html/test-ready/?setting1=const"
self.processing_url = "/data/v1/mock/html/test/?setting1=const"
self.index_url = "/data/v1/mock/html/?setting1=const"
self.empty_url = "/data/v1/mock/html/test-empty/?setting1=const"
def check_response(self, response, template, data):
self.assertIsInstance(response, TemplateResponse)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.template_name, "mock/" + template)
self.assertEqual(response.context_data, {
"self_reverse": "mock_html",
"response": data
})
def get_empty_data(self):
return copy(CommunityView.RESPONSE_DATA)
def test_html_template_for(self):
mock_response = MagicMock(spec=Response)
mock_response.status_code = 200
template = HtmlCommunityView.html_template_for(CommunityMock, mock_response)
self.assertEqual(template, "mock/ok.html")
mock_response.status_code = 202
template = HtmlCommunityView.html_template_for(CommunityMock, mock_response)
self.assertEqual(template, "mock/accepted.html")
mock_response.status_code = 204
template = HtmlCommunityView.html_template_for(CommunityMock, mock_response)
self.assertEqual(template, "mock/no-content.html")
mock_response.status_code = 404
template = HtmlCommunityView.html_template_for(CommunityMock, mock_response)
self.assertEqual(template, "mock/no-content.html")
mock_response.status_code = 400
template = HtmlCommunityView.html_template_for(CommunityMock, mock_response)
self.assertEqual(template, "mock/bad-request.html")
mock_response.status_code = 500 # or any other status
template = HtmlCommunityView.html_template_for(CommunityMock, mock_response)
self.assertEqual(template, "mock/index.html")
def test_data_for(self):
mock_data = {"test": "test"}
mock_response = MagicMock(spec=Response)
mock_response.data = mock_data
mock_response.status_code = 200
data = HtmlCommunityView.data_for(CommunityMock, mock_response)
self.assertEqual(data, mock_data)
mock_response.status_code = 300
data = HtmlCommunityView.data_for(CommunityMock, mock_response)
self.assertEqual(data, mock_data)
mock_response.status_code = 500 # or any other status
data = HtmlCommunityView.data_for(CommunityMock, mock_response)
self.assertIsNone(data)
data = HtmlCommunityView.data_for(CommunityMock, None)
self.assertIsNone(data)
@patch("core.models.organisms.community.Community.grow", side_effect=ValidationError("Invalid"))
def test_get_response_invalid(self, grow_patch):
response = self.client.get(self.processing_url)
context_data = self.get_empty_data()
context_data["error"] = "Invalid"
self.check_response(response, "bad-request.html", context_data)
self.assertContains(response, "<p>Invalid</p>")
@patch("core.models.organisms.community.Community.grow", side_effect=DSProcessUnfinished())
def test_get_response_accepted(self, grow_patch):
response = self.client.get(self.processing_url)
self.check_response(response, "accepted.html", None)
self.assertContains(response, '<p id="wait-text">')
@patch("core.models.organisms.community.Community.grow", side_effect=DSProcessError())
def test_get_response_error(self, grow_patch): # TODO: do not ignore errors
response = self.client.get(self.processing_url)
self.check_response(response, "index.html", None)
@patch("core.models.organisms.community.Community.grow", side_effect=DSProcessUnfinished())
def test_index_response(self, grow_patch):
response = self.client.get(self.index_url)
self.check_response(response, "index.html", None)
self.assertContains(response, "<p>Search for something fake.</p>")
grow_patch.assert_not_called()
CommunityMock.INPUT_THROUGH_PATH = False
response = self.client.get(self.index_url)
self.check_response(response, "accepted.html", None)
self.assertContains(response, '<p id="wait-text">')
grow_patch.assert_called_once_with("")
CommunityMock.INPUT_THROUGH_PATH = True
@patch("core.models.organisms.community.Community.grow")
def test_get_response_ok(self, grow_patch):
response = self.client.get(self.ready_url)
self.check_response(response, "ok.html", {
"error": None,
"status": {},
"result": {},
"actions": [],
"results": [
{
"_id": 4,
"id": 4,
"context": "nested value",
"value": "nested value 0",
"number": 1
},
{
"_id": 5,
"id": 5,
"context": "nested value",
"value": "nested value 1",
"number": 2
},
{
"_id": 6,
"id": 6,
"context": "nested value",
"value": "nested value 2",
"number": 3
}
]
})
self.assertContains(response, "<p>nested value 0</p>")
self.assertContains(response, "<p>nested value 1</p>")
self.assertContains(response, "<p>nested value 2</p>")
def test_get_response_empty(self):
response = self.client.get(self.empty_url)
self.check_response(response, "no-content.html", None)
self.assertContains(response, "<p>No results found, please try again.</p>")
def test_get_invalid_time(self):
response = self.client.get(self.ready_url + "&t=1")
self.assertIsInstance(response, HttpResponseNotFound)
def test_get_valid_time(self):
response = self.client.get(self.ready_url + "&t=20160605161754000")
self.assertEqual(response.status_code, 200)
self.check_response(response, "ok.html", {
"error": None,
"status": {},
"result": {},
"actions": [],
"results": [
{
"_id": 4,
"id": 4,
"context": "nested value",
"value": "nested value 0",
"number": 1
},
{
"_id": 6,
"id": 6,
"context": "nested value",
"value": "nested value 2",
"number": 3
}
]
})
self.assertContains(response, "<p>nested value 0</p>")
self.assertNotContains(response, "<p>nested value 1</p>")
self.assertContains(response, "<p>nested value 2</p>")
class TestCommunityView(TestCase):
fixtures = ["test-community"]
def setUp(self):
super().setUp()
self.view = CommunityView()
def check_response(self, response, status_code):
self.assertIsInstance(response, Response)
self.assertEqual(response.status_code, status_code)
def check_manifestations(self, count):
self.assertEqual(Manifestation.objects.count(), count)
def test_get_uri(self):
full_path = CommunityView.get_uri(CommunityMock, "test/path", {"test": "test"})
self.assertEqual(
full_path,
"/data/v1/mock/service/test/path?test=test#0dcbfb0b84aff2403ed170e77c8e80b5c2aac17586a55f03b500446a4291b105"
)
self.skipTest("add test for created_at argument")
def test_get_configuration_from_request(self):
self.skipTest("not tested")
@patch("core.models.organisms.community.Community.grow", side_effect=ValidationError("Invalid"))
def test_get_response_invalid(self, grow_patch):
response = self.view.get_response(CommunityMock, "test", {"setting1": "const"})
self.check_response(response, 400)
self.check_manifestations(0)
@patch("core.models.organisms.community.Community.grow", side_effect=DSProcessUnfinished())
def test_get_response_accepted(self, grow_patch):
response = self.view.get_response(CommunityMock, "test", {"setting1": "const"})
self.check_response(response, 202)
self.check_manifestations(0)
response = self.view.get_response(CommunityMock, "test-synchronous", {})
self.check_response(response, 202)
self.check_manifestations(0)
@patch("core.models.organisms.community.Community.grow", side_effect=DSProcessError())
def test_get_response_error(self, grow_patch):
response = self.view.get_response(CommunityMock, "test", {"setting1": "const"})
self.check_response(response, 500)
self.check_manifestations(0)
@patch("core.models.organisms.community.Community.grow")
def test_get_empty_response(self, grow_patch):
response = self.view.get_response(CommunityMock, "test-empty", {"setting1": "const"})
self.check_response(response, 204)
self.check_manifestations(1)
@patch("core.models.organisms.community.Community.grow")
def test_get_response_ok(self, grow_patch):
response = self.view.get_response(CommunityMock, "test-ready", {"setting1": "const"})
self.check_response(response, 200)
self.check_manifestations(1)
response = self.view.get_response(CommunityMock, "test-ready", {"setting1": "const"})
self.check_response(response, 200)
self.check_manifestations(1)
grow_patch.assert_called_once_with("test-ready")
def test_get(self):
client = Client()
response = client.get("/data/v1/mock/service/test-ready/?setting1=const")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {
"error": None,
"status": {},
"result": {},
"actions": [],
"results": [
{
"_id": 4,
"context": "nested value",
"value": "nested value 0",
"number": 1
},
{
"_id": 5,
"context": "nested value",
"value": "nested value 1",
"number": 2
},
{
"_id": 6,
"context": "nested value",
"value": "nested value 2",
"number": 3
}
]
})
def test_get_filter(self):
client = Client()
response = client.get("/data/v1/mock/service/test-ready/?setting1=const&include_even=0")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {
"error": None,
"status": {},
"result": {},
"actions": [],
"results": [
{
"_id": 4,
"context": "nested value",
"value": "nested value 0",
"number": 1
},
{
"_id": 6,
"context": "nested value",
"value": "nested value 2",
"number": 3
}
]
})
def test_get_invalid_time(self):
client = Client()
response = client.get("/data/v1/mock/service/test-ready/?setting1=const&t=1")
self.assertEqual(response.status_code, 404)
def test_get_valid_time(self):
client = Client()
response = client.get("/data/v1/mock/service/test-ready/?setting1=const&t=20160605161754000")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {
"error": None,
"status": {},
"result": {},
"actions": [],
"results": [
{
"_id": 4,
"context": "nested value",
"value": "nested value 0",
"number": 1
},
{
"_id": 6,
"context": "nested value",
"value": "nested value 2",
"number": 3
}
]
})
def test_post(self):
data = {
"action": "scope",
"config": {
"include_even": 0
}
}
client = Client()
response = client.post("/data/v1/mock/service/test-ready/?setting1=const",
json.dumps(data),
content_type="application/json")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json(), {
"error": None,
"status": {},
"result": {},
"actions": [],
"results": [
{
"_id": 4,
"context": "nested value",
"value": "nested value 0",
"number": 1
},
{
"_id": 6,
"context": "nested value",
"value": "nested value 2",
"number": 3
}
]
})
def raise_unfinished(self, result):
raise DSProcessUnfinished("Raised for test")
def grow(self, *args):
"""
:return:
"""
assert self.id, "A community can only be grown after an initial save."
args = args or []
if self.state == CommunityState.READY:
return True
elif self.state == CommunityState.SYNC:
return False
result = None
if self.state in [CommunityState.NEW]:
log.info("Preparing community")
self.state = CommunityState.ASYNC if self.config.asynchronous else CommunityState.SYNC
self.setup_growth(*args)
self.current_growth = self.next_growth()
self.save(
) # in between save because next operations may take long and community needs to be claimed.
log.info("Preparing " + self.current_growth.type)
self.call_begin_callback(self.current_growth.type,
self.current_growth.input)
log.info("Starting " + self.current_growth.type)
result = self.current_growth.begin(
) # when synchronous result contains actual results
self.save()
while self.kernel is None:
output, errors = self.current_growth.finish(
result) # will raise when Growth is not finished
error_count = errors.count()
if error_count > 1:
should_finish = self.call_error_callbacks(
self.current_growth.type, errors, output)
log.info("{} errors occurred".format(error_count))
else:
should_finish = True
if not should_finish:
self.state = CommunityState.ABORTED
self.save()
raise DSProcessError(
"Could not finish growth according to error callbacks.")
log.info("Finishing " + self.current_growth.type)
self.call_finish_callback(self.current_growth.type, output, errors)
try:
self.current_growth = self.next_growth()
except Growth.DoesNotExist:
self.set_kernel()
self.state = CommunityState.READY
self.save()
return True
log.info("Preparing " + self.current_growth.type)
self.call_begin_callback(self.current_growth.type,
self.current_growth.input)
log.info("Starting " + self.current_growth.type)
result = self.current_growth.begin()
self.save()
if self.state == CommunityState.ASYNC:
raise DSProcessUnfinished("Community starts another Growth.")