def test_envvars(self):
process = Process.objects.create(
name='Test environment variables',
requirements={'expression-engine': 'jinja'},
contributor=self.contributor,
type='test:data:envvars:',
input_schema=[],
output_schema=[
{'name': 'resolweapihost', 'type': 'basic:string:'},
],
run={
'language': 'bash',
'program': """
re-save resolweapihost $RESOLWE_API_HOST
"""
}
)
data = Data.objects.create(
name='Data object',
contributor=self.contributor,
process=process,
input={},
)
manager.communicate(verbosity=0)
# update output
data = Data.objects.get(pk=data.pk)
self.assertEqual(data.output['resolweapihost'], 'some.special.host')
def test_templatetags(self):
input_process = Process.objects.create(
name='Input process',
contributor=self.contributor,
type='data:test:inputobject:',
)
input_data = Data.objects.create(
name='Input Data object',
contributor=self.contributor,
process=input_process,
)
process = Process.objects.create(
name='Test template tags',
requirements={'expression-engine': 'jinja'},
contributor=self.contributor,
type='test:data:templatetags:',
input_schema=[
{'name': 'input_data', 'type': 'data:test:inputobject:'},
],
output_schema=[
{'name': 'name', 'type': 'basic:string:'},
{'name': 'id', 'type': 'basic:integer:'},
{'name': 'type', 'type': 'basic:string:'},
{'name': 'basename', 'type': 'basic:string:'},
{'name': 'subtype', 'type': 'basic:string:'},
{'name': 'yesno', 'type': 'basic:string:'},
],
run={
'language': 'bash',
'program': """
re-save name "{{ input_data | name }}"
re-save id {{ input_data | id }}
re-save type {{ input_data | type }}
re-save basename "{{ '/foo/bar/moo' | basename }}"
re-save subtype "{{ 'data:test:inputobject:' | subtype('data:') }}"
re-save yesno "{{ true | yesno('yes', 'no') }}"
"""
}
)
data = Data.objects.create(
name='Data object',
contributor=self.contributor,
process=process,
input={'input_data': input_data.pk},
)
manager.communicate(verbosity=0)
# update output
data = Data.objects.get(pk=data.pk)
self.assertEqual(data.output['name'], input_data.name)
self.assertEqual(data.output['id'], input_data.pk)
self.assertEqual(data.output['type'], input_process.type)
self.assertEqual(data.output['basename'], 'moo')
self.assertEqual(data.output['subtype'], 'True')
self.assertEqual(data.output['yesno'], 'yes')
def test_envvars(self):
flow_executor = copy.copy(getattr(settings, 'FLOW_EXECUTOR', {}))
flow_executor['SET_ENV'] = {
'SET_ENV_TEST': 'test_var',
}
with override_settings(FLOW_EXECUTOR=flow_executor):
process = Process.objects.create(
name='Test environment variables',
requirements={'expression-engine': 'jinja'},
contributor=self.contributor,
type='test:data:envvars:',
input_schema=[],
output_schema=[
{
'name': 'resolweapihost',
'type': 'basic:string:'
},
{
'name': 'setenvtest',
'type': 'basic:string:'
},
],
run={
'language':
'bash',
'program':
"""
re-save resolweapihost $RESOLWE_HOST_URL
re-save setenvtest $SET_ENV_TEST
"""
})
data = Data.objects.create(
name='Data object',
contributor=self.contributor,
process=process,
input={},
)
manager.communicate(verbosity=0)
# update output
data = Data.objects.get(pk=data.pk)
self.assertEqual(data.output['resolweapihost'],
'some.special.host')
self.assertEqual(data.output['setenvtest'], 'test_var')
def manager_post_save_handler(sender, instance, created, **kwargs):
"""Run newly created (spawned) processes."""
if instance.status == Data.STATUS_DONE or instance.status == Data.STATUS_ERROR or created:
# Run manager at the end of the potential transaction. Otherwise
# tasks are send to workers before transaction ends and therefore
# workers cannot access objects created inside transaction.
transaction.on_commit(lambda: manager.communicate(verbosity=0))
def test_workflow(self):
self.run_process('test-workflow-1', {'param1': 'world'},
run_manager=False)
workflow_data = Data.objects.get(process__slug='test-workflow-1')
# Assign permissions before manager is called
assign_perm('view_data', self.contributor, workflow_data)
assign_perm('view_data', self.group, workflow_data)
# One manager call for each created object
manager.communicate(run_sync=True, verbosity=0)
manager.communicate(run_sync=True, verbosity=0)
manager.communicate(run_sync=True, verbosity=0)
workflow_data.refresh_from_db()
step1_data = Data.objects.get(process__slug='test-example-1')
step2_data = Data.objects.get(process__slug='test-example-2')
# Workflow should output indices of all data objects, in order.
self.assertEqual(workflow_data.output['steps'],
[step1_data.pk, step2_data.pk])
# Steps should execute with the correct variables.
self.assertEqual(step1_data.input['param1'], 'world')
self.assertEqual(step1_data.input['param2'], True)
self.assertEqual(step1_data.output['out1'], 'hello world')
self.assertEqual(step2_data.input['param1'], step1_data.pk)
self.assertEqual(step2_data.input['param2']['a'], step1_data.pk)
self.assertEqual(step2_data.input['param2']['b'], 'hello')
self.assertEqual(step2_data.output['out1'], 'simon says: hello world')
self.assertEqual(step1_data.parents.count(), 1)
self.assertEqual(step1_data.parents.first(), workflow_data)
self.assertEqual(step2_data.parents.count(), 2)
six.assertCountEqual(self, step2_data.parents.all(),
[workflow_data, step1_data])
# Check correct dependency type is created.
self.assertEqual({d.kind
for d in step1_data.parents_dependency.all()},
{DataDependency.KIND_SUBPROCESS})
self.assertEqual(
{d.kind
for d in step2_data.parents_dependency.all()},
{DataDependency.KIND_SUBPROCESS, DataDependency.KIND_IO})
self.assertTrue(self.contributor.has_perm('flow.view_data',
step1_data))
# User inherites permission from group
self.assertTrue(self.user.has_perm('flow.view_data', step1_data))
def run_process(self, process_slug, input_={}, assert_status=Data.STATUS_DONE,
descriptor=None, descriptor_schema=None, run_manager=True,
verbosity=0):
"""Runs given processor with specified inputs.
If input is file, file path should be given relative to
``tests/files`` folder of a Django application.
If ``assert_status`` is given check if Data object's status
matches ``assert_status`` after finishing processor.
:param processor_name: name of the processor to run
:type processor_name: :obj:`str`
:param ``input_``: Input paramaters for processor. You don't
have to specifie parameters for which default values are
given.
:type ``input_``: :obj:`dict`
:param ``assert_status``: Desired status of Data object
:type ``assert_status``: :obj:`str`
:param descriptor: Descriptor to set on the data object.
:type descriptor: :obj:`dict`
:param descriptor_schema: Descriptor schema to set on the data object.
:type descriptor_schema: :obj:`dict`
:return: :obj:`resolwe.flow.models.Data` object which is created by
the processor.
"""
# backward compatibility
process_slug = slugify(process_slug.replace(':', '-'))
p = Process.objects.get(slug=process_slug)
def mock_upload(file_path):
old_path = os.path.join(self.files_path, file_path)
if not os.path.isfile(old_path):
raise RuntimeError('Missing file: {}'.format(old_path))
new_path = os.path.join(self.upload_dir, file_path)
# create directories needed by new_path
new_path_dir = os.path.dirname(new_path)
if not os.path.exists(new_path_dir):
os.makedirs(new_path_dir)
shutil.copy2(old_path, new_path)
self._upload_files.append(new_path)
return {
'file': file_path,
'file_temp': file_path,
}
for field_schema, fields in iterate_fields(input_, p.input_schema):
# copy referenced files to upload dir
if field_schema['type'] == "basic:file:":
fields[field_schema['name']] = mock_upload(fields[field_schema['name']])
elif field_schema['type'] == "list:basic:file:":
file_list = [mock_upload(file_path) for file_path in fields[field_schema['name']]]
fields[field_schema['name']] = file_list
# convert primary keys to strings
if field_schema['type'].startswith('data:'):
fields[field_schema['name']] = fields[field_schema['name']]
if field_schema['type'].startswith('list:data:'):
fields[field_schema['name']] = [obj for obj in fields[field_schema['name']]]
d = Data.objects.create(
input=input_,
contributor=self.admin,
process=p,
slug=get_random_string(length=6),
descriptor_schema=descriptor_schema,
descriptor=descriptor or {})
self.collection.data.add(d)
if run_manager:
manager.communicate(run_sync=True, verbosity=verbosity)
# Fetch latest Data object from database
d = Data.objects.get(pk=d.pk)
if not run_manager and assert_status == Data.STATUS_DONE:
assert_status = Data.STATUS_RESOLVING
if assert_status:
self.assertStatus(d, assert_status)
return d
def test_templatetags(self):
input_process = Process.objects.create(name='Input process',
contributor=self.contributor,
type='data:test:inputobject:',
output_schema=[
{
'name': 'test_file',
'type': 'basic:file:'
},
],
run={
'language':
'bash',
'program':
"""
mkdir -p path/to
touch path/to/file.txt
re-save-file test_file path/to/file.txt
"""
})
descriptor_schema = DescriptorSchema.objects.create(
name='Test schema',
slug='test-schema',
contributor=self.contributor,
schema=[{
'name': 'descriptions',
'required': False,
'group': [
{
'name': 'text',
'type': 'basic:string:'
},
]
}])
input_data = Data.objects.create(
name='Input Data object',
contributor=self.contributor,
process=input_process,
descriptor_schema=descriptor_schema,
descriptor={'descriptions': {
'text': 'This is test Data object.'
}})
manager.communicate(verbosity=0, run_sync=True)
process = Process.objects.create(
name='Test template tags',
requirements={'expression-engine': 'jinja'},
contributor=self.contributor,
type='test:data:templatetags:',
input_schema=[
{
'name': 'input_data',
'type': 'data:test:inputobject:'
},
{
'name': 'input_data_list',
'type': 'list:data:test:inputobject:'
},
{
'name': 'spacy',
'type': 'basic:string:'
},
],
output_schema=[
{
'name': 'name',
'type': 'basic:string:'
},
{
'name': 'list_name',
'type': 'list:basic:string:'
},
{
'name': 'id',
'type': 'basic:integer:'
},
{
'name': 'list_id',
'type': 'list:basic:string:'
},
{
'name': 'type',
'type': 'basic:string:'
},
{
'name': 'list_type',
'type': 'list:basic:string:'
},
{
'name': 'basename',
'type': 'basic:string:'
},
{
'name': 'subtype',
'type': 'basic:string:'
},
{
'name': 'list_subtype',
'type': 'list:basic:string:'
},
{
'name': 'yesno',
'type': 'basic:string:'
},
{
'name': 'datalookup',
'type': 'basic:integer:'
},
{
'name': 'file_url',
'type': 'basic:string:'
},
{
'name': 'unsafe',
'type': 'basic:string:'
},
{
'name': 'safe',
'type': 'basic:string:'
},
{
'name': 'description_text',
'type': 'basic:string:'
},
{
'name': 'description_full',
'type': 'basic:json:'
},
{
'name': 'list_description_text',
'type': 'basic:string:'
},
{
'name': 'list_description_full',
'type': 'basic:json:'
},
{
'name': 'all',
'type': 'basic:string:'
},
{
'name': 'any',
'type': 'basic:string:'
},
],
run={
'language':
'bash',
'program':
"""
re-save name {{ input_data | name }}
re-save list_name {{ input_data_list | name }}
re-save id {{ input_data | id }}
re-save list_id {{ input_data_list | id }}
re-save type {{ input_data | type }}
re-save list_type {{ input_data_list | type }}
re-save basename {{ '/foo/bar/moo' | basename }}
re-save subtype {{ 'data:test:inputobject:' | subtype('data:') }}
re-save list_subtype {{ ['data:test:inputobject:'] | subtype('data:') }}
re-save yesno {{ true | yesno('yes', 'no') }}
re-save datalookup {{ 'input-data-object' | data_by_slug }}
re-save file_url {{ input_data.test_file | get_url }}
re-save unsafe {{ spacy }}
re-save description_text {{ input_data | descriptor('descriptions.text') }}
re-save description_full {{ input_data | descriptor }}
re-save list_description_text {{ input_data_list[0] | descriptor('descriptions.text') }}
re-save list_description_full {{ input_data_list[0] | descriptor }}
re-save all {{ [true, false] | all }}
re-save any {{ [true, false] | any }}
function save-safe() {
re-save safe $1
}
save-safe {{ spacy | safe }}
"""
})
data = Data.objects.create(
name='Data object',
contributor=self.contributor,
process=process,
input={
'input_data': input_data.pk,
'input_data_list': [input_data.pk],
'spacy': 'this has \'some\' spaces',
},
)
manager.communicate(verbosity=0, run_sync=True)
data.refresh_from_db()
self.assertEqual(data.output['name'], input_data.name)
self.assertEqual(data.output['id'], input_data.pk)
self.assertEqual(data.output['list_id'], [input_data.id])
self.assertEqual(data.output['type'], input_process.type)
self.assertEqual(data.output['basename'], 'moo')
self.assertEqual(data.output['subtype'], 'True')
self.assertEqual(data.output['yesno'], 'yes')
self.assertEqual(data.output['datalookup'], input_data.pk)
self.assertEqual(
data.output['file_url'],
'localhost/data/{}/path/to/file.txt'.format(input_data.pk))
self.assertEqual(data.output['unsafe'], 'this has \'some\' spaces')
self.assertEqual(data.output['safe'], 'this')
self.assertEqual(data.output['description_text'],
'This is test Data object.')
self.assertEqual(data.output['list_description_text'],
'This is test Data object.')
storage = Storage.objects.get(pk=data.output['description_full'])
self.assertEqual(
storage.json,
{'descriptions': {
'text': 'This is test Data object.'
}})
storage = Storage.objects.get(pk=data.output['list_description_full'])
self.assertEqual(
storage.json,
{'descriptions': {
'text': 'This is test Data object.'
}})
# Return values of jinja filters are casted to strings when inserted to bash
self.assertEqual(data.output['list_name'], str([input_data.name]))
self.assertEqual(data.output['list_type'], str([input_process.type]))
self.assertEqual(data.output['list_subtype'], '[True]')
self.assertEqual(data.output['all'], 'False')
self.assertEqual(data.output['any'], 'True')
def manager_post_save_handler(sender, instance, **kwargs):
if instance.status == Data.STATUS_DONE or instance.status == Data.STATUS_ERROR:
manager.communicate()
def test_manager(self):
manager.communicate(verbosity=0)
def manager_post_save_handler(sender, instance, **kwargs):
"""Run newly created (spawned) processes."""
if instance.status == Data.STATUS_DONE or instance.status == Data.STATUS_ERROR:
manager.communicate(verbosity=0)
def test_templatetags(self):
input_process = Process.objects.create(
name='Input process',
contributor=self.contributor,
type='data:test:inputobject:',
)
input_data = Data.objects.create(
name='Input Data object',
contributor=self.contributor,
process=input_process,
)
process = Process.objects.create(
name='Test template tags',
requirements={'expression-engine': 'jinja'},
contributor=self.contributor,
type='test:data:templatetags:',
input_schema=[
{
'name': 'input_data',
'type': 'data:test:inputobject:'
},
],
output_schema=[
{
'name': 'name',
'type': 'basic:string:'
},
{
'name': 'id',
'type': 'basic:integer:'
},
{
'name': 'type',
'type': 'basic:string:'
},
{
'name': 'basename',
'type': 'basic:string:'
},
{
'name': 'subtype',
'type': 'basic:string:'
},
{
'name': 'yesno',
'type': 'basic:string:'
},
{
'name': 'datalookup',
'type': 'basic:integer:'
},
],
run={
'language':
'bash',
'program':
"""
re-save name "{{ input_data | name }}"
re-save id {{ input_data | id }}
re-save type {{ input_data | type }}
re-save basename "{{ '/foo/bar/moo' | basename }}"
re-save subtype "{{ 'data:test:inputobject:' | subtype('data:') }}"
re-save yesno "{{ true | yesno('yes', 'no') }}"
re-save datalookup "{{ 'input-data-object' | data_by_slug }}"
"""
})
data = Data.objects.create(
name='Data object',
contributor=self.contributor,
process=process,
input={'input_data': input_data.pk},
)
manager.communicate(verbosity=0)
# update output
data = Data.objects.get(pk=data.pk)
self.assertEqual(data.output['name'], input_data.name)
self.assertEqual(data.output['id'], input_data.pk)
self.assertEqual(data.output['type'], input_process.type)
self.assertEqual(data.output['basename'], 'moo')
self.assertEqual(data.output['subtype'], 'True')
self.assertEqual(data.output['yesno'], 'yes')
self.assertEqual(data.output['datalookup'], input_data.pk)
def run_process(self, process_slug, input_={}, assert_status=Data.STATUS_DONE, run_manager=True, verbosity=0):
"""Runs given processor with specified inputs.
If input is file, file path should be given relative to
``tests/files`` folder of a Django application.
If ``assert_status`` is given check if Data object's status
matches ``assert_status`` after finishing processor.
:param processor_name: name of the processor to run
:type processor_name: :obj:`str`
:param ``input_``: Input paramaters for processor. You don't
have to specifie parameters for which default values are
given.
:type ``input_``: :obj:`dict`
:param ``assert_status``: Desired status of Data object
:type ``assert_status``: :obj:`str`
:return: :obj:`resolwe.flow.models.Data` object which is created by
the processor.
"""
# backward compatibility
process_slug = slugify(process_slug.replace(':', '-'))
p = Process.objects.get(slug=process_slug)
for field_schema, fields in iterate_fields(input_, p.input_schema):
# copy referenced files to upload dir
if field_schema['type'] == "basic:file:":
for app_config in apps.get_app_configs():
old_path = os.path.join(app_config.path, 'tests', 'files', fields[field_schema['name']])
if os.path.isfile(old_path):
file_name = os.path.basename(fields[field_schema['name']])
new_path = os.path.join(self.upload_path, file_name)
shutil.copy2(old_path, new_path)
self._upload_files.append(new_path)
# since we don't know what uid/gid will be used inside Docker executor,
# we must give others read and write permissions
os.chmod(new_path, 0o666)
fields[field_schema['name']] = {
'file': file_name,
'file_temp': file_name,
}
break
# convert primary keys to strings
if field_schema['type'].startswith('data:'):
fields[field_schema['name']] = str(fields[field_schema['name']])
if field_schema['type'].startswith('list:data:'):
fields[field_schema['name']] = [str(obj) for obj in fields[field_schema['name']]]
d = Data.objects.create(
input=input_,
contributor=self.admin,
process=p,
slug=get_random_string(length=6))
self.collection.data.add(d)
if run_manager:
manager.communicate(run_sync=True, verbosity=verbosity)
# Fetch latest Data object from database
d = Data.objects.get(pk=d.pk)
if not run_manager and assert_status == Data.STATUS_DONE:
assert_status = Data.STATUS_RESOLVING
if assert_status:
self.assertStatus(d, assert_status)
return d
def manager_post_save_handler(sender, instance, **kwargs):
"""Run newly created (spawned) processes."""
if instance.status == Data.STATUS_DONE or instance.status == Data.STATUS_ERROR:
manager.communicate(verbosity=0)
def commit_signal():
"""Nudge manager at the end of every Data object save event."""
if not getattr(settings, 'FLOW_MANAGER_DISABLE_AUTO_CALLS', False):
immediate = getattr(settings, 'FLOW_MANAGER_SYNC_AUTO_CALLS', False)
manager.communicate(verbosity=0, save_settings=False, run_sync=immediate)
def run_process(self, process_slug, input_={}, assert_status=Data.STATUS_DONE,
descriptor=None, descriptor_schema=None, run_manager=True,
verbosity=0):
"""Run the specified process with the given inputs.
If input is a file, file path should be given relative to the
``tests/files`` directory of a Django application.
If ``assert_status`` is given, check if
:class:`~resolwe.flow.models.Data` object's status matches
it after the process has finished.
:param str process_slug: slug of the
:class:`~resolwe.flow.models.Process` to run
:param dict ``input_``: :class:`~resolwe.flow.models.Process`'s
input parameters
.. note::
You don't have to specify parameters with defined
default values.
:param str ``assert_status``: desired status of the
:class:`~resolwe.flow.models.Data` object
:param dict descriptor: descriptor to set on the
:class:`~resolwe.flow.models.Data` object
:param dict descriptor_schema: descriptor schema to set on the
:class:`~resolwe.flow.models.Data` object
:return: object created by
:class:`~resolwe.flow.models.Process`
:rtype: ~resolwe.flow.models.Data
"""
# backward compatibility
process_slug = slugify(process_slug.replace(':', '-'))
process = Process.objects.filter(slug=process_slug).order_by('-version').first()
def mock_upload(file_path):
"""Mock file upload."""
old_path = os.path.join(self.files_path, file_path)
if not os.path.isfile(old_path):
raise RuntimeError('Missing file: {}'.format(old_path))
new_path = os.path.join(self.upload_dir, file_path)
# create directories needed by new_path
new_path_dir = os.path.dirname(new_path)
if not os.path.exists(new_path_dir):
os.makedirs(new_path_dir)
shutil.copy2(old_path, new_path)
self._upload_files.append(new_path)
return {
'file': file_path,
'file_temp': file_path,
}
for field_schema, fields in iterate_fields(input_, process.input_schema):
# copy referenced files to upload dir
if field_schema['type'] == "basic:file:":
fields[field_schema['name']] = mock_upload(fields[field_schema['name']])
elif field_schema['type'] == "list:basic:file:":
file_list = [mock_upload(file_path) for file_path in fields[field_schema['name']]]
fields[field_schema['name']] = file_list
# convert primary keys to strings
if field_schema['type'].startswith('data:'):
fields[field_schema['name']] = fields[field_schema['name']]
if field_schema['type'].startswith('list:data:'):
fields[field_schema['name']] = [obj for obj in fields[field_schema['name']]]
data = Data.objects.create(
input=input_,
contributor=self.admin,
process=process,
slug=get_random_string(length=6),
descriptor_schema=descriptor_schema,
descriptor=descriptor or {})
self.collection.data.add(data)
if run_manager:
manager.communicate(run_sync=True, verbosity=verbosity)
# Fetch latest Data object from database
data = Data.objects.get(pk=data.pk)
if not run_manager and assert_status == Data.STATUS_DONE:
assert_status = Data.STATUS_RESOLVING
if assert_status:
self.assertStatus(data, assert_status)
return data
def run_process(self, process_slug, input_={}, assert_status=Data.STATUS_DONE,
descriptor=None, descriptor_schema=None, run_manager=True,
verbosity=0):
"""Run the specified process with the given inputs.
If input is a file, file path should be given relative to the
``tests/files`` directory of a Django application.
If ``assert_status`` is given, check if
:class:`~resolwe.flow.models.Data` object's status matches
it after the process has finished.
:param str process_slug: slug of the
:class:`~resolwe.flow.models.Process` to run
:param dict ``input_``: :class:`~resolwe.flow.models.Process`'s
input parameters
.. note::
You don't have to specify parameters with defined
default values.
:param str ``assert_status``: desired status of the
:class:`~resolwe.flow.models.Data` object
:param dict descriptor: descriptor to set on the
:class:`~resolwe.flow.models.Data` object
:param dict descriptor_schema: descriptor schema to set on the
:class:`~resolwe.flow.models.Data` object
:return: object created by
:class:`~resolwe.flow.models.Process`
:rtype: ~resolwe.flow.models.Data
"""
# backward compatibility
process_slug = slugify(process_slug.replace(':', '-'))
process = Process.objects.filter(slug=process_slug).order_by('-version').first()
def mock_upload(file_path):
"""Mock file upload."""
old_path = os.path.join(self.files_path, file_path)
if not os.path.isfile(old_path):
raise RuntimeError('Missing file: {}'.format(old_path))
file_temp = '{}_{}'.format(file_path, uuid.uuid4())
upload_file_path = os.path.join(self.upload_dir, file_temp)
# create directories needed by new_path
upload_file_dir = os.path.dirname(upload_file_path)
if not os.path.exists(upload_file_dir):
os.makedirs(upload_file_dir)
shutil.copy2(old_path, upload_file_path)
self._upload_files.append(upload_file_path)
return {
'file': file_path,
'file_temp': file_temp,
}
for field_schema, fields in iterate_fields(input_, process.input_schema):
# copy referenced files to upload dir
if field_schema['type'] == "basic:file:":
fields[field_schema['name']] = mock_upload(fields[field_schema['name']])
elif field_schema['type'] == "list:basic:file:":
file_list = [mock_upload(file_path) for file_path in fields[field_schema['name']]]
fields[field_schema['name']] = file_list
# convert primary keys to strings
if field_schema['type'].startswith('data:'):
fields[field_schema['name']] = fields[field_schema['name']]
if field_schema['type'].startswith('list:data:'):
fields[field_schema['name']] = [obj for obj in fields[field_schema['name']]]
data = Data.objects.create(
input=input_,
contributor=self.admin,
process=process,
slug=get_random_string(length=6),
descriptor_schema=descriptor_schema,
descriptor=descriptor or {})
self.collection.data.add(data)
if run_manager:
manager.communicate(run_sync=True, verbosity=verbosity)
# Fetch latest Data object from database
data = Data.objects.get(pk=data.pk)
if not run_manager and assert_status == Data.STATUS_DONE:
assert_status = Data.STATUS_RESOLVING
if assert_status:
self.assertStatus(data, assert_status)
return data
def test_name(self):
process = Process.objects.create(slug='test-first',
type='test:first',
contributor=self.user,
data_name='Process based data name',
output_schema=[{
'name': 'stat',
'type': 'basic:string:',
}],
run={'bash': 'echo {"stat": "42"}'})
data = Data.objects.create(contributor=self.user,
process=process)
self.assertEqual(data.name, 'Process based data name')
self.assertFalse(data.named_by_user)
data.name = 'Name changed by user'
data.save()
self.assertEqual(data.name, 'Name changed by user')
self.assertTrue(data.named_by_user)
data = Data.objects.create(name='Explicit data name',
contributor=self.user,
process=process)
self.assertEqual(data.name, 'Explicit data name')
self.assertTrue(data.named_by_user)
process = Process.objects.create(slug='test-second',
type='test:second',
contributor=self.user,
data_name='Process based data name, value: {{src.stat}}',
input_schema=[{
'name': 'src',
'type': 'data:test:first:',
'required': False,
}])
second = Data.objects.create(contributor=self.user,
process=process,
input={'src': data.id})
data.output = {'stat': '42'}
data.status = 'OK'
data.save()
self.assertEqual(second.name, 'Process based data name, value: ')
self.assertFalse(second.named_by_user)
manager.communicate(verbosity=0)
second = Data.objects.get(id=second.id)
self.assertEqual(second.name, 'Process based data name, value: 42')
self.assertFalse(second.named_by_user)
data.output = {}
data.status = 'RE'
data.save()
second = Data.objects.create(contributor=self.user,
process=process,
input={'src': data.id})
second.name = 'User\' data name'
second.save()
data.output = {'stat': '42'}
data.status = 'OK'
data.save()
self.assertEqual(second.name, 'User\' data name')
self.assertTrue(second.named_by_user)
manager.communicate(verbosity=0)
second = Data.objects.get(id=second.id)
self.assertEqual(second.name, 'User\' data name')
self.assertTrue(second.named_by_user)
def test_name(self):
process = Process.objects.create(slug='test-first',
type='data:test:first:',
contributor=self.contributor,
data_name='Process based data name',
output_schema=[{
'name': 'stat',
'type': 'basic:string:',
'required': False,
}],
run={
'language': 'bash',
'program': 'echo {"stat": "42"}'
})
data = Data.objects.create(contributor=self.contributor,
process=process)
self.assertEqual(data.name, 'Process based data name')
self.assertFalse(data.named_by_user)
data.name = 'Name changed by user'
data.save()
self.assertEqual(data.name, 'Name changed by user')
self.assertTrue(data.named_by_user)
data = Data.objects.create(name='Explicit data name',
contributor=self.contributor,
process=process)
self.assertEqual(data.name, 'Explicit data name')
self.assertTrue(data.named_by_user)
process = Process.objects.create(
slug='test-second',
type='test:second',
contributor=self.contributor,
requirements={'expression-engine': 'jinja'},
data_name='Process based data name, value: {{src.stat}}',
input_schema=[{
'name': 'src',
'type': 'data:test:first:',
'required': False,
}])
second = Data.objects.create(contributor=self.contributor,
process=process,
input={'src': data.id})
data.output = {'stat': '42'}
data.status = 'OK'
data.save()
self.assertEqual(second.name, 'Process based data name, value: ')
self.assertFalse(second.named_by_user)
manager.communicate(verbosity=0)
second = Data.objects.get(id=second.id)
self.assertEqual(second.name, 'Process based data name, value: 42')
self.assertFalse(second.named_by_user)
data.output = {}
data.status = 'RE'
data.save()
second = Data.objects.create(contributor=self.contributor,
process=process,
input={'src': data.id})
second.name = 'User\' data name'
second.save()
data.output = {'stat': '42'}
data.status = 'OK'
data.save()
self.assertEqual(second.name, 'User\' data name')
self.assertTrue(second.named_by_user)
manager.communicate(verbosity=0)
second = Data.objects.get(id=second.id)
self.assertEqual(second.name, 'User\' data name')
self.assertTrue(second.named_by_user)
def test_manager(self):
manager.communicate(verbosity=0)
def create(self, request, *args, **kwargs):
"""Create a resource."""
collections = request.data.get('collections', [])
# check that user has permissions on all collections that Data
# object will be added to
for collection_id in collections:
try:
collection = Collection.objects.get(pk=collection_id)
except Collection.DoesNotExist:
return Response(
{
'collections': [
'Invalid pk "{}" - object does not exist.'.format(
collection_id)
]
},
status=status.HTTP_400_BAD_REQUEST)
if not request.user.has_perm('add_collection', obj=collection):
if request.user.has_perm('view_collection', obj=collection):
raise exceptions.PermissionDenied(
"You don't have `ADD` permission on collection (id: {})."
.format(collection_id))
else:
raise exceptions.NotFound(
"Collection not found (id: {}).".format(collection_id))
# translate processe's slug to id
process_slug = request.data.get('process', None)
process_query = Process.objects.filter(slug=process_slug)
process_query = get_objects_for_user(request.user, 'view_process',
process_query)
try:
process = process_query.latest()
except Process.DoesNotExist:
return Response(
{
'process': [
'Invalid process slug "{}" - object does not exist.'.
format(process_slug)
]
},
status=status.HTTP_400_BAD_REQUEST)
request.data['process'] = process.pk
# perform "get_or_create" if requested - return existing object
# if found
if kwargs.pop('get_or_create', False):
process_input = request.data.get('input', {})
# use default values if they are not given
for field_schema, fields, path in iterate_schema(
process_input, process.input_schema):
if 'default' in field_schema and field_schema[
'name'] not in fields:
dict_dot(process_input, path, field_schema['default'])
checksum = get_data_checksum(process_input, process.slug,
process.version)
data_qs = Data.objects.filter(
checksum=checksum,
process__persistence__in=[
Process.PERSISTENCE_CACHED, Process.PERSISTENCE_TEMP
],
)
data_qs = get_objects_for_user(request.user, 'view_data', data_qs)
if data_qs.exists():
data = data_qs.order_by('created').last()
serializer = self.get_serializer(data)
return Response(serializer.data)
# create the objects
resp = super(DataViewSet, self).create(request, *args, **kwargs)
# run manager
manager.communicate()
return resp
def test_templatetags(self):
input_process = Process.objects.create(name='Input process',
contributor=self.contributor,
type='data:test:inputobject:',
output_schema=[
{
'name': 'test_file',
'type': 'basic:file:'
},
],
run={
'language':
'bash',
'program':
"""
mkdir -p path/to
touch path/to/file.txt
re-save-file test_file path/to/file.txt
"""
})
input_data = Data.objects.create(
name='Input Data object',
contributor=self.contributor,
process=input_process,
)
manager.communicate(verbosity=0)
process = Process.objects.create(
name='Test template tags',
requirements={'expression-engine': 'jinja'},
contributor=self.contributor,
type='test:data:templatetags:',
input_schema=[
{
'name': 'input_data',
'type': 'data:test:inputobject:'
},
],
output_schema=[
{
'name': 'name',
'type': 'basic:string:'
},
{
'name': 'id',
'type': 'basic:integer:'
},
{
'name': 'type',
'type': 'basic:string:'
},
{
'name': 'basename',
'type': 'basic:string:'
},
{
'name': 'subtype',
'type': 'basic:string:'
},
{
'name': 'yesno',
'type': 'basic:string:'
},
{
'name': 'datalookup',
'type': 'basic:integer:'
},
{
'name': 'file_url',
'type': 'basic:string:'
},
],
run={
'language':
'bash',
'program':
"""
re-save name "{{ input_data | name }}"
re-save id {{ input_data | id }}
re-save type {{ input_data | type }}
re-save basename "{{ '/foo/bar/moo' | basename }}"
re-save subtype "{{ 'data:test:inputobject:' | subtype('data:') }}"
re-save yesno "{{ true | yesno('yes', 'no') }}"
re-save datalookup "{{ 'input-data-object' | data_by_slug }}"
re-save file_url "{{ input_data.test_file | get_url }}"
"""
})
data = Data.objects.create(
name='Data object',
contributor=self.contributor,
process=process,
input={'input_data': input_data.pk},
)
manager.communicate(verbosity=0)
data.refresh_from_db()
self.assertEqual(data.output['name'], input_data.name)
self.assertEqual(data.output['id'], input_data.pk)
self.assertEqual(data.output['type'], input_process.type)
self.assertEqual(data.output['basename'], 'moo')
self.assertEqual(data.output['subtype'], 'True')
self.assertEqual(data.output['yesno'], 'yes')
self.assertEqual(data.output['datalookup'], input_data.pk)
self.assertEqual(
data.output['file_url'],
'localhost/data/{}/path/to/file.txt'.format(input_data.pk))
