def testCreate_file(self):
data = fixtures.data_objects.file_data_object
s = DataObjectSerializer(data=data)
s.is_valid(raise_exception=True)
data_object = s.save()
self.assertEqual(data_object.file_resource.filename,
data['value']['filename'])
def testCreate_file(self):
data = fixtures.data_objects.file_data_object
s = DataObjectSerializer(data=data)
s.is_valid(raise_exception=True)
data_object = s.save()
self.assertEqual(data_object.file_resource.filename,
data['value']['filename'])
def testCreate_nonFileTypes(self):
for data in self.data_object_fixtures:
s = DataObjectSerializer(data=data)
s.is_valid(raise_exception=True)
data_object = s.save()
self.assertEqual(data_object.value, data['value'])
rendered_data = DataObjectSerializer(
data_object, context=get_mock_context()).data
self.assertEqual(data_object.value, rendered_data['value'])
def testUpdateUploadStatus(self):
file_data = fixtures.data_objects.file_data_object['value']
data_object = DataObject.create_and_initialize_file_resource(**file_data)
s = DataObjectSerializer(data_object,
context=get_mock_context())
s.save()
s2 = DataObjectUpdateSerializer(data_object)
s2.update(
data_object, {'value': {'upload_status': 'error'}})
self.assertEqual(s2.data['value']['upload_status'], 'error')
def testUpdateProtectedValue(self):
file_data = fixtures.data_objects.file_data_object['value']
data_object = DataObject.create_and_initialize_file_resource(**file_data)
s = DataObjectSerializer(data_object,
context=get_mock_context())
s.save()
s2 = DataObjectUpdateSerializer(data_object)
with self.assertRaises(ValidationError):
s2.update(
data_object, {'type': 'string'})
def testCreate_noDroolOnFail(self):
file_data = copy.deepcopy(fixtures.data_objects.file_data_object)
file_data['value']['md5'] = 'invalid_md5'
data_object_count_before = DataObject.objects.count()
s = DataObjectSerializer(data=file_data)
s.is_valid(raise_exception=True)
with self.assertRaises(ValidationError):
m = s.save()
data_object_count_after = DataObject.objects.count()
self.assertEqual(data_object_count_before, data_object_count_after)
def testCreate_noDroolOnFail(self):
file_data = copy.deepcopy(fixtures.data_objects.file_data_object)
file_data['value']['md5'] = 'invalid_md5'
data_object_count_before = DataObject.objects.count()
s = DataObjectSerializer(data=file_data)
s.is_valid(raise_exception=True)
with self.assertRaises(ValidationError):
m = s.save()
data_object_count_after = DataObject.objects.count()
self.assertEqual(data_object_count_before, data_object_count_after)
def testCreate_AlreadyExists(self):
file_data = copy.deepcopy(fixtures.data_objects.file_data_object)['value']
data_object = DataObject.create_and_initialize_file_resource(**file_data)
s = DataObjectSerializer(data_object,
context=get_mock_context())
rendered_1 = s.data
data_object_count_before = DataObject.objects.count()
s = DataObjectSerializer(data=rendered_1)
s.is_valid(raise_exception=True)
data_object = s.save()
# Verify that no new object was created
data_object_count_after = DataObject.objects.count()
self.assertEqual(data_object_count_before, data_object_count_after)
class TaskAttemptLogFileSerializer(CreateWithParentModelSerializer):
class Meta:
model = TaskAttemptLogFile
fields = ('uuid', 'url', 'log_name', 'data_object')
uuid = serializers.CharField(required=False)
url = serializers.HyperlinkedIdentityField(
view_name='task-attempt-log-file-detail', lookup_field='uuid')
data_object = DataObjectSerializer(allow_null=True, required=False)
def update(self, instance, validated_data):
data_object_data = self.initial_data.get('data_object', None)
validated_data.pop('data_object', None)
if data_object_data:
if data_object_data.get('type') \
and not data_object_data.get('type') == 'file':
raise serializers.ValidationError(
'Bad type "%s". Must be type "file".' %
data_object_data.get('type'))
if instance.data_object:
raise serializers.ValidationError(
'Updating existing nested file not allowed')
s = DataObjectSerializer(data=dataobject_data,
context={
'request': self.request,
'task_attempt': instance.task_attempt
})
s.is_valid(raise_exception=True)
s.save()
return instance
def testCreate_ErrorAlreadyExistsWithMismatch(self):
file_data = copy.deepcopy(fixtures.data_objects.file_data_object)['value']
data_object = DataObject.create_and_initialize_file_resource(**file_data)
s = DataObjectSerializer(data_object,
context=get_mock_context())
data_object_count_before = DataObject.objects.count()
rendered_1 = s.data
rendered_1['value']['md5'] = '192f08c86f675deca469ea50ffac38e0'
s = DataObjectSerializer(data=rendered_1)
with self.assertRaises(ValidationError):
s.is_valid(raise_exception=True)
data_object = s.save()
# Verify that no new object was created
data_object_count_after = DataObject.objects.count()
self.assertEqual(data_object_count_before, data_object_count_after)
def testRender_file(self):
file_data = fixtures.data_objects.file_data_object['value']
data_object = DataObject.create_and_initialize_file_resource(**file_data)
s = DataObjectSerializer(data_object,
context=get_mock_context())
rendered_data = s.data
value = rendered_data['value']
self.assertEqual(value['filename'], file_data['filename'])
self.assertEqual(value['md5'], file_data['md5'])
self.assertEqual(value['source_type'], file_data['source_type'])
self.assertEqual(value['import_comments'], file_data['import_comments'])
self.assertEqual(value['imported_from_url'], file_data['imported_from_url'])
def testCreateReference(self):
value = True
s = DataObjectSerializer(data={'type': 'boolean', 'value': value})
s.is_valid(raise_exception=True)
do = s.save()
contents = {'uuid': do.uuid,
'url': 'http://127.0.0.1/data-objects/%s/' % do.uuid}
s = DataNodeSerializer(
data={'contents': contents},
context={'type': 'boolean'})
s.is_valid(raise_exception=True)
m = s.save()
self.assertEqual(m.data_object.value, value)
def update(self, instance, validated_data):
data_object_data = self.initial_data.get('data_object', None)
validated_data.pop('data_object', None)
if data_object_data:
if data_object_data.get('type') \
and not data_object_data.get('type') == 'file':
raise serializers.ValidationError(
'Bad type "%s". Must be type "file".' %
data_object_data.get('type'))
if instance.data_object:
raise serializers.ValidationError(
'Updating existing nested file not allowed')
s = DataObjectSerializer(data=dataobject_data,
context={
'request': self.request,
'task_attempt': instance.task_attempt}
)
s.is_valid(raise_exception=True)
s.save()
return instance
def testCreate_errorUuidCollision(self):
file_data = copy.deepcopy(fixtures.data_objects.file_data_object)['value']
data_object = DataObject.create_and_initialize_file_resource(**file_data)
s = DataObjectSerializer(data_object,
context=get_mock_context())
rendered_1 = s.data
s = DataObjectSerializer(data=rendered_1)
with self.assertRaises(ValidationError):
s.is_valid(raise_exception=True)
data_object = s.save()
def testRoundTrip_nonFileTypes(self):
for data in self.data_object_fixtures:
s1 = DataObjectSerializer(data=data)
s1.is_valid(raise_exception=True)
data_object_1 = s1.save()
self.assertEqual(data_object_1.value, data['value'])
rendered_data = DataObjectSerializer(
data_object_1, context=get_mock_context()).data
# Update UUID to avoid collision
rendered_data['uuid'] = uuid.uuid4()
s2 = DataObjectSerializer(data=rendered_data)
s2.is_valid(raise_exception=True)
data_object_2 = s2.save()
self.assertEqual(data_object_1.value, data_object_2.value)
def testCreate_nonFileTypes(self):
for data in self.data_object_fixtures:
s = DataObjectSerializer(data=data)
s.is_valid(raise_exception=True)
data_object = s.save()
self.assertEqual(data_object.value, data['value'])
rendered_data = DataObjectSerializer(
data_object, context=get_mock_context()).data
self.assertEqual(data_object.value, rendered_data['value'])
def testRoundTrip_file(self):
file_data = fixtures.data_objects.file_data_object['value']
data_object = DataObject.create_and_initialize_file_resource(**file_data)
s = DataObjectSerializer(data_object,
context=get_mock_context())
rendered_1 = s.data
# update UUID to avoid collision
input_2 = copy.deepcopy(rendered_1)
input_2['uuid'] = str(uuid.uuid4())
s = DataObjectSerializer(data=input_2)
s.is_valid(raise_exception=True)
data_object = s.save()
s = DataObjectSerializer(data_object,
context=get_mock_context())
rendered_2 = s.data
self.assertEqual(rendered_1['type'],
rendered_2['type'])
self.assertEqual(rendered_1['datetime_created'],
rendered_2['datetime_created'])
self.assertNotEqual(rendered_1['uuid'],
rendered_2['uuid'])
self.assertNotEqual(rendered_1['url'],
rendered_2['url'])
self.assertEqual(rendered_1['value']['filename'],
rendered_2['value']['filename'])
self.assertEqual(rendered_1['value']['md5'],
rendered_2['value']['md5'])
self.assertEqual(rendered_1['value']['import_comments'],
rendered_2['value']['import_comments'])
self.assertEqual(rendered_1['value']['imported_from_url'],
rendered_2['value']['imported_from_url'])
self.assertEqual(rendered_1['value']['upload_status'],
rendered_2['value']['upload_status'])
self.assertEqual(rendered_1['value']['source_type'],
rendered_2['value']['source_type'])
self.assertEqual(rendered_1['value']['file_url'],
rendered_2['value']['file_url'])
def update(self, instance, validated_data):
data_object_data = self.initial_data.get('data_object', None)
validated_data.pop('data_object', None)
if data_object_data:
if data_object_data.get('type') \
and not data_object_data.get('type') == 'file':
raise serializers.ValidationError(
'Bad type "%s". Must be type "file".' %
data_object_data.get('type'))
if instance.data_object:
raise serializers.ValidationError(
'Updating existing nested file not allowed')
s = DataObjectSerializer(data=dataobject_data,
context={
'request': self.request,
'task_attempt': instance.task_attempt
})
s.is_valid(raise_exception=True)
s.save()
return instance
def testRoundTrip_file(self):
file_data = fixtures.data_objects.file_data_object['value']
data_object = DataObject.create_and_initialize_file_resource(**file_data)
s = DataObjectSerializer(data_object,
context=get_mock_context())
rendered_1 = s.data
# update UUID to avoid collision
input_2 = copy.deepcopy(rendered_1)
input_2['uuid'] = str(uuid.uuid4())
s = DataObjectSerializer(data=input_2)
s.is_valid(raise_exception=True)
data_object = s.save()
s = DataObjectSerializer(data_object,
context=get_mock_context())
rendered_2 = s.data
self.assertEqual(rendered_1['type'],
rendered_2['type'])
self.assertEqual(rendered_1['datetime_created'],
rendered_2['datetime_created'])
self.assertNotEqual(rendered_1['uuid'],
rendered_2['uuid'])
self.assertNotEqual(rendered_1['url'],
rendered_2['url'])
self.assertEqual(rendered_1['value']['filename'],
rendered_2['value']['filename'])
self.assertEqual(rendered_1['value']['md5'],
rendered_2['value']['md5'])
self.assertEqual(rendered_1['value']['import_comments'],
rendered_2['value']['import_comments'])
self.assertEqual(rendered_1['value']['imported_from_url'],
rendered_2['value']['imported_from_url'])
self.assertEqual(rendered_1['value']['upload_status'],
rendered_2['value']['upload_status'])
self.assertEqual(rendered_1['value']['source_type'],
rendered_2['value']['source_type'])
self.assertEqual(rendered_1['value']['file_url'],
rendered_2['value']['file_url'])
def testCreate_countQueries(self):
data = fixtures.data_objects.file_data_object
s = DataObjectSerializer(data=data)
s.is_valid(raise_exception=True)
self.assertNumQueries(4, lambda: s.save())
def get_task():
task = Task.objects.create(interpreter='/bin/bash',
raw_command='echo {{input1}}',
command='echo True',
resources={
'memory': '1',
'disk_size': '1',
'cores': '1'
},
environment={'docker_image': 'ubuntu'},
data_path=[
[0, 1],
])
input_data_object = DataObject.objects.create(type='boolean',
data={'value': True})
input_data_node = create_data_node_from_data_object(input_data_object)
input = TaskInput.objects.create(task=task,
data_node=input_data_node,
channel='input1',
mode='no_gather',
type='boolean')
output_data_object = DataObject.objects.create(type='string',
data={'value': 'answer'})
output_data_node = create_data_node_from_data_object(output_data_object)
task_output = TaskOutput.objects.create(task=task,
channel='output1',
type='string',
mode='no_scatter',
data_node=output_data_node,
source={'stream': 'stdout'})
attempt_output_data_node = create_data_node_from_data_object(
output_data_object)
task_attempt = TaskAttempt.objects.create(task=task,
interpreter=task.interpreter,
command=task.command,
environment=task.environment,
resources=task.resources)
task_attempt_output = TaskAttemptOutput.objects.create(
task_attempt=task_attempt,
data_node=attempt_output_data_node,
mode='no_scatter',
type='file',
channel='test')
task_attempt_event = TaskAttemptEvent.objects.create(
task_attempt=task_attempt,
event='oops',
detail='something went wrong',
is_error=True)
log_file = TaskAttemptLogFile.objects.create(
task_attempt=task_attempt,
log_name='stderr.log',
)
file_data = {
'type': 'file',
'value': {
'filename': 'stderr.log',
'md5': 'eed7ca1bba1c93a7fa5b5dba1307b791',
'source_type': 'log',
}
}
s = DataObjectSerializer(data=file_data,
context={'task_attempt_log_file': log_file})
s.is_valid()
log_file_data_object = s.save()
return task
def get_task():
task = Task.objects.create(
interpreter='/bin/bash',
raw_command='echo {{input1}}',
command='echo True',
resources={'memory': '1', 'disk_size': '1', 'cores': '1'},
environment={'docker_image': 'ubuntu'},
data_path=[[0,1],]
)
input_data_object = DataObject.objects.create(
type='boolean',
data={'value': True}
)
input_data_node = create_data_node_from_data_object(input_data_object)
input = TaskInput.objects.create(task=task,
data_node=input_data_node,
channel='input1',
mode='no_gather',
type='boolean')
output_data_object = DataObject.objects.create(
type='string',
data={'value': 'answer'}
)
output_data_node = create_data_node_from_data_object(output_data_object)
task_output = TaskOutput.objects.create(
task=task,
channel='output1',
type='string',
mode='no_scatter',
data_node=output_data_node,
source={'stream': 'stdout'}
)
attempt_output_data_node = create_data_node_from_data_object(output_data_object)
task_attempt = TaskAttempt.objects.create(
interpreter = task.interpreter,
command=task.command,
environment=task.environment,
resources=task.resources)
task.add_to_all_task_attempts(task_attempt)
task_attempt_output = TaskAttemptOutput.objects.create(
task_attempt=task_attempt,
data_node=attempt_output_data_node,
mode='no_scatter',
type='file',
channel='test'
)
task_attempt_event = TaskAttemptEvent.objects.create(
task_attempt=task_attempt,
event='oops',
detail='something went wrong',
is_error=True)
log_file = TaskAttemptLogFile.objects.create(
task_attempt=task_attempt,
log_name='stderr.log',
)
file_data = {
'type': 'file',
'value': {
'filename': 'stderr.log',
'md5': 'eed7ca1bba1c93a7fa5b5dba1307b791',
'source_type': 'log',
}
}
s = DataObjectSerializer(
data=file_data,
context={'task_attempt_log_file': log_file})
s.is_valid()
log_file_data_object = s.save()
return task
def testRoundTrip_nonFileTypes(self):
for data in self.data_object_fixtures:
s1 = DataObjectSerializer(data=data)
s1.is_valid(raise_exception=True)
data_object_1 = s1.save()
self.assertEqual(data_object_1.value, data['value'])
rendered_data = DataObjectSerializer(
data_object_1, context=get_mock_context()).data
# Update UUID to avoid collision
rendered_data['uuid'] = uuid.uuid4()
s2 = DataObjectSerializer(data=rendered_data)
s2.is_valid(raise_exception=True)
data_object_2 = s2.save()
self.assertEqual(data_object_1.value, data_object_2.value)