def test_envvars(self): process = Process.objects.create( name='Test environment variables', requirements={'expression-engine': 'jinja'}, contributor=self.contributor, type='test:data:envvars:', input_schema=[], output_schema=[ {'name': 'resolweapihost', 'type': 'basic:string:'}, ], run={ 'language': 'bash', 'program': """ re-save resolweapihost $RESOLWE_API_HOST """ } ) data = Data.objects.create( name='Data object', contributor=self.contributor, process=process, input={}, ) manager.communicate(verbosity=0) # update output data = Data.objects.get(pk=data.pk) self.assertEqual(data.output['resolweapihost'], 'some.special.host')
def test_templatetags(self): input_process = Process.objects.create( name='Input process', contributor=self.contributor, type='data:test:inputobject:', ) input_data = Data.objects.create( name='Input Data object', contributor=self.contributor, process=input_process, ) process = Process.objects.create( name='Test template tags', requirements={'expression-engine': 'jinja'}, contributor=self.contributor, type='test:data:templatetags:', input_schema=[ {'name': 'input_data', 'type': 'data:test:inputobject:'}, ], output_schema=[ {'name': 'name', 'type': 'basic:string:'}, {'name': 'id', 'type': 'basic:integer:'}, {'name': 'type', 'type': 'basic:string:'}, {'name': 'basename', 'type': 'basic:string:'}, {'name': 'subtype', 'type': 'basic:string:'}, {'name': 'yesno', 'type': 'basic:string:'}, ], run={ 'language': 'bash', 'program': """ re-save name "{{ input_data | name }}" re-save id {{ input_data | id }} re-save type {{ input_data | type }} re-save basename "{{ '/foo/bar/moo' | basename }}" re-save subtype "{{ 'data:test:inputobject:' | subtype('data:') }}" re-save yesno "{{ true | yesno('yes', 'no') }}" """ } ) data = Data.objects.create( name='Data object', contributor=self.contributor, process=process, input={'input_data': input_data.pk}, ) manager.communicate(verbosity=0) # update output data = Data.objects.get(pk=data.pk) self.assertEqual(data.output['name'], input_data.name) self.assertEqual(data.output['id'], input_data.pk) self.assertEqual(data.output['type'], input_process.type) self.assertEqual(data.output['basename'], 'moo') self.assertEqual(data.output['subtype'], 'True') self.assertEqual(data.output['yesno'], 'yes')
def test_envvars(self): flow_executor = copy.copy(getattr(settings, 'FLOW_EXECUTOR', {})) flow_executor['SET_ENV'] = { 'SET_ENV_TEST': 'test_var', } with override_settings(FLOW_EXECUTOR=flow_executor): process = Process.objects.create( name='Test environment variables', requirements={'expression-engine': 'jinja'}, contributor=self.contributor, type='test:data:envvars:', input_schema=[], output_schema=[ { 'name': 'resolweapihost', 'type': 'basic:string:' }, { 'name': 'setenvtest', 'type': 'basic:string:' }, ], run={ 'language': 'bash', 'program': """ re-save resolweapihost $RESOLWE_HOST_URL re-save setenvtest $SET_ENV_TEST """ }) data = Data.objects.create( name='Data object', contributor=self.contributor, process=process, input={}, ) manager.communicate(verbosity=0) # update output data = Data.objects.get(pk=data.pk) self.assertEqual(data.output['resolweapihost'], 'some.special.host') self.assertEqual(data.output['setenvtest'], 'test_var')
def manager_post_save_handler(sender, instance, created, **kwargs): """Run newly created (spawned) processes.""" if instance.status == Data.STATUS_DONE or instance.status == Data.STATUS_ERROR or created: # Run manager at the end of the potential transaction. Otherwise # tasks are send to workers before transaction ends and therefore # workers cannot access objects created inside transaction. transaction.on_commit(lambda: manager.communicate(verbosity=0))
def test_workflow(self): self.run_process('test-workflow-1', {'param1': 'world'}, run_manager=False) workflow_data = Data.objects.get(process__slug='test-workflow-1') # Assign permissions before manager is called assign_perm('view_data', self.contributor, workflow_data) assign_perm('view_data', self.group, workflow_data) # One manager call for each created object manager.communicate(run_sync=True, verbosity=0) manager.communicate(run_sync=True, verbosity=0) manager.communicate(run_sync=True, verbosity=0) workflow_data.refresh_from_db() step1_data = Data.objects.get(process__slug='test-example-1') step2_data = Data.objects.get(process__slug='test-example-2') # Workflow should output indices of all data objects, in order. self.assertEqual(workflow_data.output['steps'], [step1_data.pk, step2_data.pk]) # Steps should execute with the correct variables. self.assertEqual(step1_data.input['param1'], 'world') self.assertEqual(step1_data.input['param2'], True) self.assertEqual(step1_data.output['out1'], 'hello world') self.assertEqual(step2_data.input['param1'], step1_data.pk) self.assertEqual(step2_data.input['param2']['a'], step1_data.pk) self.assertEqual(step2_data.input['param2']['b'], 'hello') self.assertEqual(step2_data.output['out1'], 'simon says: hello world') self.assertEqual(step1_data.parents.count(), 1) self.assertEqual(step1_data.parents.first(), workflow_data) self.assertEqual(step2_data.parents.count(), 2) six.assertCountEqual(self, step2_data.parents.all(), [workflow_data, step1_data]) # Check correct dependency type is created. self.assertEqual({d.kind for d in step1_data.parents_dependency.all()}, {DataDependency.KIND_SUBPROCESS}) self.assertEqual( {d.kind for d in step2_data.parents_dependency.all()}, {DataDependency.KIND_SUBPROCESS, DataDependency.KIND_IO}) self.assertTrue(self.contributor.has_perm('flow.view_data', step1_data)) # User inherites permission from group self.assertTrue(self.user.has_perm('flow.view_data', step1_data))
def run_process(self, process_slug, input_={}, assert_status=Data.STATUS_DONE, descriptor=None, descriptor_schema=None, run_manager=True, verbosity=0): """Runs given processor with specified inputs. If input is file, file path should be given relative to ``tests/files`` folder of a Django application. If ``assert_status`` is given check if Data object's status matches ``assert_status`` after finishing processor. :param processor_name: name of the processor to run :type processor_name: :obj:`str` :param ``input_``: Input paramaters for processor. You don't have to specifie parameters for which default values are given. :type ``input_``: :obj:`dict` :param ``assert_status``: Desired status of Data object :type ``assert_status``: :obj:`str` :param descriptor: Descriptor to set on the data object. :type descriptor: :obj:`dict` :param descriptor_schema: Descriptor schema to set on the data object. :type descriptor_schema: :obj:`dict` :return: :obj:`resolwe.flow.models.Data` object which is created by the processor. """ # backward compatibility process_slug = slugify(process_slug.replace(':', '-')) p = Process.objects.get(slug=process_slug) def mock_upload(file_path): old_path = os.path.join(self.files_path, file_path) if not os.path.isfile(old_path): raise RuntimeError('Missing file: {}'.format(old_path)) new_path = os.path.join(self.upload_dir, file_path) # create directories needed by new_path new_path_dir = os.path.dirname(new_path) if not os.path.exists(new_path_dir): os.makedirs(new_path_dir) shutil.copy2(old_path, new_path) self._upload_files.append(new_path) return { 'file': file_path, 'file_temp': file_path, } for field_schema, fields in iterate_fields(input_, p.input_schema): # copy referenced files to upload dir if field_schema['type'] == "basic:file:": fields[field_schema['name']] = mock_upload(fields[field_schema['name']]) elif field_schema['type'] == "list:basic:file:": file_list = [mock_upload(file_path) for file_path in fields[field_schema['name']]] fields[field_schema['name']] = file_list # convert primary keys to strings if field_schema['type'].startswith('data:'): fields[field_schema['name']] = fields[field_schema['name']] if field_schema['type'].startswith('list:data:'): fields[field_schema['name']] = [obj for obj in fields[field_schema['name']]] d = Data.objects.create( input=input_, contributor=self.admin, process=p, slug=get_random_string(length=6), descriptor_schema=descriptor_schema, descriptor=descriptor or {}) self.collection.data.add(d) if run_manager: manager.communicate(run_sync=True, verbosity=verbosity) # Fetch latest Data object from database d = Data.objects.get(pk=d.pk) if not run_manager and assert_status == Data.STATUS_DONE: assert_status = Data.STATUS_RESOLVING if assert_status: self.assertStatus(d, assert_status) return d
def test_templatetags(self): input_process = Process.objects.create(name='Input process', contributor=self.contributor, type='data:test:inputobject:', output_schema=[ { 'name': 'test_file', 'type': 'basic:file:' }, ], run={ 'language': 'bash', 'program': """ mkdir -p path/to touch path/to/file.txt re-save-file test_file path/to/file.txt """ }) descriptor_schema = DescriptorSchema.objects.create( name='Test schema', slug='test-schema', contributor=self.contributor, schema=[{ 'name': 'descriptions', 'required': False, 'group': [ { 'name': 'text', 'type': 'basic:string:' }, ] }]) input_data = Data.objects.create( name='Input Data object', contributor=self.contributor, process=input_process, descriptor_schema=descriptor_schema, descriptor={'descriptions': { 'text': 'This is test Data object.' }}) manager.communicate(verbosity=0, run_sync=True) process = Process.objects.create( name='Test template tags', requirements={'expression-engine': 'jinja'}, contributor=self.contributor, type='test:data:templatetags:', input_schema=[ { 'name': 'input_data', 'type': 'data:test:inputobject:' }, { 'name': 'input_data_list', 'type': 'list:data:test:inputobject:' }, { 'name': 'spacy', 'type': 'basic:string:' }, ], output_schema=[ { 'name': 'name', 'type': 'basic:string:' }, { 'name': 'list_name', 'type': 'list:basic:string:' }, { 'name': 'id', 'type': 'basic:integer:' }, { 'name': 'list_id', 'type': 'list:basic:string:' }, { 'name': 'type', 'type': 'basic:string:' }, { 'name': 'list_type', 'type': 'list:basic:string:' }, { 'name': 'basename', 'type': 'basic:string:' }, { 'name': 'subtype', 'type': 'basic:string:' }, { 'name': 'list_subtype', 'type': 'list:basic:string:' }, { 'name': 'yesno', 'type': 'basic:string:' }, { 'name': 'datalookup', 'type': 'basic:integer:' }, { 'name': 'file_url', 'type': 'basic:string:' }, { 'name': 'unsafe', 'type': 'basic:string:' }, { 'name': 'safe', 'type': 'basic:string:' }, { 'name': 'description_text', 'type': 'basic:string:' }, { 'name': 'description_full', 'type': 'basic:json:' }, { 'name': 'list_description_text', 'type': 'basic:string:' }, { 'name': 'list_description_full', 'type': 'basic:json:' }, { 'name': 'all', 'type': 'basic:string:' }, { 'name': 'any', 'type': 'basic:string:' }, ], run={ 'language': 'bash', 'program': """ re-save name {{ input_data | name }} re-save list_name {{ input_data_list | name }} re-save id {{ input_data | id }} re-save list_id {{ input_data_list | id }} re-save type {{ input_data | type }} re-save list_type {{ input_data_list | type }} re-save basename {{ '/foo/bar/moo' | basename }} re-save subtype {{ 'data:test:inputobject:' | subtype('data:') }} re-save list_subtype {{ ['data:test:inputobject:'] | subtype('data:') }} re-save yesno {{ true | yesno('yes', 'no') }} re-save datalookup {{ 'input-data-object' | data_by_slug }} re-save file_url {{ input_data.test_file | get_url }} re-save unsafe {{ spacy }} re-save description_text {{ input_data | descriptor('descriptions.text') }} re-save description_full {{ input_data | descriptor }} re-save list_description_text {{ input_data_list[0] | descriptor('descriptions.text') }} re-save list_description_full {{ input_data_list[0] | descriptor }} re-save all {{ [true, false] | all }} re-save any {{ [true, false] | any }} function save-safe() { re-save safe $1 } save-safe {{ spacy | safe }} """ }) data = Data.objects.create( name='Data object', contributor=self.contributor, process=process, input={ 'input_data': input_data.pk, 'input_data_list': [input_data.pk], 'spacy': 'this has \'some\' spaces', }, ) manager.communicate(verbosity=0, run_sync=True) data.refresh_from_db() self.assertEqual(data.output['name'], input_data.name) self.assertEqual(data.output['id'], input_data.pk) self.assertEqual(data.output['list_id'], [input_data.id]) self.assertEqual(data.output['type'], input_process.type) self.assertEqual(data.output['basename'], 'moo') self.assertEqual(data.output['subtype'], 'True') self.assertEqual(data.output['yesno'], 'yes') self.assertEqual(data.output['datalookup'], input_data.pk) self.assertEqual( data.output['file_url'], 'localhost/data/{}/path/to/file.txt'.format(input_data.pk)) self.assertEqual(data.output['unsafe'], 'this has \'some\' spaces') self.assertEqual(data.output['safe'], 'this') self.assertEqual(data.output['description_text'], 'This is test Data object.') self.assertEqual(data.output['list_description_text'], 'This is test Data object.') storage = Storage.objects.get(pk=data.output['description_full']) self.assertEqual( storage.json, {'descriptions': { 'text': 'This is test Data object.' }}) storage = Storage.objects.get(pk=data.output['list_description_full']) self.assertEqual( storage.json, {'descriptions': { 'text': 'This is test Data object.' }}) # Return values of jinja filters are casted to strings when inserted to bash self.assertEqual(data.output['list_name'], str([input_data.name])) self.assertEqual(data.output['list_type'], str([input_process.type])) self.assertEqual(data.output['list_subtype'], '[True]') self.assertEqual(data.output['all'], 'False') self.assertEqual(data.output['any'], 'True')
def manager_post_save_handler(sender, instance, **kwargs): if instance.status == Data.STATUS_DONE or instance.status == Data.STATUS_ERROR: manager.communicate()
def test_manager(self): manager.communicate(verbosity=0)
def manager_post_save_handler(sender, instance, **kwargs): """Run newly created (spawned) processes.""" if instance.status == Data.STATUS_DONE or instance.status == Data.STATUS_ERROR: manager.communicate(verbosity=0)
def test_templatetags(self): input_process = Process.objects.create( name='Input process', contributor=self.contributor, type='data:test:inputobject:', ) input_data = Data.objects.create( name='Input Data object', contributor=self.contributor, process=input_process, ) process = Process.objects.create( name='Test template tags', requirements={'expression-engine': 'jinja'}, contributor=self.contributor, type='test:data:templatetags:', input_schema=[ { 'name': 'input_data', 'type': 'data:test:inputobject:' }, ], output_schema=[ { 'name': 'name', 'type': 'basic:string:' }, { 'name': 'id', 'type': 'basic:integer:' }, { 'name': 'type', 'type': 'basic:string:' }, { 'name': 'basename', 'type': 'basic:string:' }, { 'name': 'subtype', 'type': 'basic:string:' }, { 'name': 'yesno', 'type': 'basic:string:' }, { 'name': 'datalookup', 'type': 'basic:integer:' }, ], run={ 'language': 'bash', 'program': """ re-save name "{{ input_data | name }}" re-save id {{ input_data | id }} re-save type {{ input_data | type }} re-save basename "{{ '/foo/bar/moo' | basename }}" re-save subtype "{{ 'data:test:inputobject:' | subtype('data:') }}" re-save yesno "{{ true | yesno('yes', 'no') }}" re-save datalookup "{{ 'input-data-object' | data_by_slug }}" """ }) data = Data.objects.create( name='Data object', contributor=self.contributor, process=process, input={'input_data': input_data.pk}, ) manager.communicate(verbosity=0) # update output data = Data.objects.get(pk=data.pk) self.assertEqual(data.output['name'], input_data.name) self.assertEqual(data.output['id'], input_data.pk) self.assertEqual(data.output['type'], input_process.type) self.assertEqual(data.output['basename'], 'moo') self.assertEqual(data.output['subtype'], 'True') self.assertEqual(data.output['yesno'], 'yes') self.assertEqual(data.output['datalookup'], input_data.pk)
def run_process(self, process_slug, input_={}, assert_status=Data.STATUS_DONE, run_manager=True, verbosity=0): """Runs given processor with specified inputs. If input is file, file path should be given relative to ``tests/files`` folder of a Django application. If ``assert_status`` is given check if Data object's status matches ``assert_status`` after finishing processor. :param processor_name: name of the processor to run :type processor_name: :obj:`str` :param ``input_``: Input paramaters for processor. You don't have to specifie parameters for which default values are given. :type ``input_``: :obj:`dict` :param ``assert_status``: Desired status of Data object :type ``assert_status``: :obj:`str` :return: :obj:`resolwe.flow.models.Data` object which is created by the processor. """ # backward compatibility process_slug = slugify(process_slug.replace(':', '-')) p = Process.objects.get(slug=process_slug) for field_schema, fields in iterate_fields(input_, p.input_schema): # copy referenced files to upload dir if field_schema['type'] == "basic:file:": for app_config in apps.get_app_configs(): old_path = os.path.join(app_config.path, 'tests', 'files', fields[field_schema['name']]) if os.path.isfile(old_path): file_name = os.path.basename(fields[field_schema['name']]) new_path = os.path.join(self.upload_path, file_name) shutil.copy2(old_path, new_path) self._upload_files.append(new_path) # since we don't know what uid/gid will be used inside Docker executor, # we must give others read and write permissions os.chmod(new_path, 0o666) fields[field_schema['name']] = { 'file': file_name, 'file_temp': file_name, } break # convert primary keys to strings if field_schema['type'].startswith('data:'): fields[field_schema['name']] = str(fields[field_schema['name']]) if field_schema['type'].startswith('list:data:'): fields[field_schema['name']] = [str(obj) for obj in fields[field_schema['name']]] d = Data.objects.create( input=input_, contributor=self.admin, process=p, slug=get_random_string(length=6)) self.collection.data.add(d) if run_manager: manager.communicate(run_sync=True, verbosity=verbosity) # Fetch latest Data object from database d = Data.objects.get(pk=d.pk) if not run_manager and assert_status == Data.STATUS_DONE: assert_status = Data.STATUS_RESOLVING if assert_status: self.assertStatus(d, assert_status) return d
def commit_signal(): """Nudge manager at the end of every Data object save event.""" if not getattr(settings, 'FLOW_MANAGER_DISABLE_AUTO_CALLS', False): immediate = getattr(settings, 'FLOW_MANAGER_SYNC_AUTO_CALLS', False) manager.communicate(verbosity=0, save_settings=False, run_sync=immediate)
def run_process(self, process_slug, input_={}, assert_status=Data.STATUS_DONE, descriptor=None, descriptor_schema=None, run_manager=True, verbosity=0): """Run the specified process with the given inputs. If input is a file, file path should be given relative to the ``tests/files`` directory of a Django application. If ``assert_status`` is given, check if :class:`~resolwe.flow.models.Data` object's status matches it after the process has finished. :param str process_slug: slug of the :class:`~resolwe.flow.models.Process` to run :param dict ``input_``: :class:`~resolwe.flow.models.Process`'s input parameters .. note:: You don't have to specify parameters with defined default values. :param str ``assert_status``: desired status of the :class:`~resolwe.flow.models.Data` object :param dict descriptor: descriptor to set on the :class:`~resolwe.flow.models.Data` object :param dict descriptor_schema: descriptor schema to set on the :class:`~resolwe.flow.models.Data` object :return: object created by :class:`~resolwe.flow.models.Process` :rtype: ~resolwe.flow.models.Data """ # backward compatibility process_slug = slugify(process_slug.replace(':', '-')) process = Process.objects.filter(slug=process_slug).order_by('-version').first() def mock_upload(file_path): """Mock file upload.""" old_path = os.path.join(self.files_path, file_path) if not os.path.isfile(old_path): raise RuntimeError('Missing file: {}'.format(old_path)) new_path = os.path.join(self.upload_dir, file_path) # create directories needed by new_path new_path_dir = os.path.dirname(new_path) if not os.path.exists(new_path_dir): os.makedirs(new_path_dir) shutil.copy2(old_path, new_path) self._upload_files.append(new_path) return { 'file': file_path, 'file_temp': file_path, } for field_schema, fields in iterate_fields(input_, process.input_schema): # copy referenced files to upload dir if field_schema['type'] == "basic:file:": fields[field_schema['name']] = mock_upload(fields[field_schema['name']]) elif field_schema['type'] == "list:basic:file:": file_list = [mock_upload(file_path) for file_path in fields[field_schema['name']]] fields[field_schema['name']] = file_list # convert primary keys to strings if field_schema['type'].startswith('data:'): fields[field_schema['name']] = fields[field_schema['name']] if field_schema['type'].startswith('list:data:'): fields[field_schema['name']] = [obj for obj in fields[field_schema['name']]] data = Data.objects.create( input=input_, contributor=self.admin, process=process, slug=get_random_string(length=6), descriptor_schema=descriptor_schema, descriptor=descriptor or {}) self.collection.data.add(data) if run_manager: manager.communicate(run_sync=True, verbosity=verbosity) # Fetch latest Data object from database data = Data.objects.get(pk=data.pk) if not run_manager and assert_status == Data.STATUS_DONE: assert_status = Data.STATUS_RESOLVING if assert_status: self.assertStatus(data, assert_status) return data
def run_process(self, process_slug, input_={}, assert_status=Data.STATUS_DONE, descriptor=None, descriptor_schema=None, run_manager=True, verbosity=0): """Run the specified process with the given inputs. If input is a file, file path should be given relative to the ``tests/files`` directory of a Django application. If ``assert_status`` is given, check if :class:`~resolwe.flow.models.Data` object's status matches it after the process has finished. :param str process_slug: slug of the :class:`~resolwe.flow.models.Process` to run :param dict ``input_``: :class:`~resolwe.flow.models.Process`'s input parameters .. note:: You don't have to specify parameters with defined default values. :param str ``assert_status``: desired status of the :class:`~resolwe.flow.models.Data` object :param dict descriptor: descriptor to set on the :class:`~resolwe.flow.models.Data` object :param dict descriptor_schema: descriptor schema to set on the :class:`~resolwe.flow.models.Data` object :return: object created by :class:`~resolwe.flow.models.Process` :rtype: ~resolwe.flow.models.Data """ # backward compatibility process_slug = slugify(process_slug.replace(':', '-')) process = Process.objects.filter(slug=process_slug).order_by('-version').first() def mock_upload(file_path): """Mock file upload.""" old_path = os.path.join(self.files_path, file_path) if not os.path.isfile(old_path): raise RuntimeError('Missing file: {}'.format(old_path)) file_temp = '{}_{}'.format(file_path, uuid.uuid4()) upload_file_path = os.path.join(self.upload_dir, file_temp) # create directories needed by new_path upload_file_dir = os.path.dirname(upload_file_path) if not os.path.exists(upload_file_dir): os.makedirs(upload_file_dir) shutil.copy2(old_path, upload_file_path) self._upload_files.append(upload_file_path) return { 'file': file_path, 'file_temp': file_temp, } for field_schema, fields in iterate_fields(input_, process.input_schema): # copy referenced files to upload dir if field_schema['type'] == "basic:file:": fields[field_schema['name']] = mock_upload(fields[field_schema['name']]) elif field_schema['type'] == "list:basic:file:": file_list = [mock_upload(file_path) for file_path in fields[field_schema['name']]] fields[field_schema['name']] = file_list # convert primary keys to strings if field_schema['type'].startswith('data:'): fields[field_schema['name']] = fields[field_schema['name']] if field_schema['type'].startswith('list:data:'): fields[field_schema['name']] = [obj for obj in fields[field_schema['name']]] data = Data.objects.create( input=input_, contributor=self.admin, process=process, slug=get_random_string(length=6), descriptor_schema=descriptor_schema, descriptor=descriptor or {}) self.collection.data.add(data) if run_manager: manager.communicate(run_sync=True, verbosity=verbosity) # Fetch latest Data object from database data = Data.objects.get(pk=data.pk) if not run_manager and assert_status == Data.STATUS_DONE: assert_status = Data.STATUS_RESOLVING if assert_status: self.assertStatus(data, assert_status) return data
def test_name(self): process = Process.objects.create(slug='test-first', type='test:first', contributor=self.user, data_name='Process based data name', output_schema=[{ 'name': 'stat', 'type': 'basic:string:', }], run={'bash': 'echo {"stat": "42"}'}) data = Data.objects.create(contributor=self.user, process=process) self.assertEqual(data.name, 'Process based data name') self.assertFalse(data.named_by_user) data.name = 'Name changed by user' data.save() self.assertEqual(data.name, 'Name changed by user') self.assertTrue(data.named_by_user) data = Data.objects.create(name='Explicit data name', contributor=self.user, process=process) self.assertEqual(data.name, 'Explicit data name') self.assertTrue(data.named_by_user) process = Process.objects.create(slug='test-second', type='test:second', contributor=self.user, data_name='Process based data name, value: {{src.stat}}', input_schema=[{ 'name': 'src', 'type': 'data:test:first:', 'required': False, }]) second = Data.objects.create(contributor=self.user, process=process, input={'src': data.id}) data.output = {'stat': '42'} data.status = 'OK' data.save() self.assertEqual(second.name, 'Process based data name, value: ') self.assertFalse(second.named_by_user) manager.communicate(verbosity=0) second = Data.objects.get(id=second.id) self.assertEqual(second.name, 'Process based data name, value: 42') self.assertFalse(second.named_by_user) data.output = {} data.status = 'RE' data.save() second = Data.objects.create(contributor=self.user, process=process, input={'src': data.id}) second.name = 'User\' data name' second.save() data.output = {'stat': '42'} data.status = 'OK' data.save() self.assertEqual(second.name, 'User\' data name') self.assertTrue(second.named_by_user) manager.communicate(verbosity=0) second = Data.objects.get(id=second.id) self.assertEqual(second.name, 'User\' data name') self.assertTrue(second.named_by_user)
def test_name(self): process = Process.objects.create(slug='test-first', type='data:test:first:', contributor=self.contributor, data_name='Process based data name', output_schema=[{ 'name': 'stat', 'type': 'basic:string:', 'required': False, }], run={ 'language': 'bash', 'program': 'echo {"stat": "42"}' }) data = Data.objects.create(contributor=self.contributor, process=process) self.assertEqual(data.name, 'Process based data name') self.assertFalse(data.named_by_user) data.name = 'Name changed by user' data.save() self.assertEqual(data.name, 'Name changed by user') self.assertTrue(data.named_by_user) data = Data.objects.create(name='Explicit data name', contributor=self.contributor, process=process) self.assertEqual(data.name, 'Explicit data name') self.assertTrue(data.named_by_user) process = Process.objects.create( slug='test-second', type='test:second', contributor=self.contributor, requirements={'expression-engine': 'jinja'}, data_name='Process based data name, value: {{src.stat}}', input_schema=[{ 'name': 'src', 'type': 'data:test:first:', 'required': False, }]) second = Data.objects.create(contributor=self.contributor, process=process, input={'src': data.id}) data.output = {'stat': '42'} data.status = 'OK' data.save() self.assertEqual(second.name, 'Process based data name, value: ') self.assertFalse(second.named_by_user) manager.communicate(verbosity=0) second = Data.objects.get(id=second.id) self.assertEqual(second.name, 'Process based data name, value: 42') self.assertFalse(second.named_by_user) data.output = {} data.status = 'RE' data.save() second = Data.objects.create(contributor=self.contributor, process=process, input={'src': data.id}) second.name = 'User\' data name' second.save() data.output = {'stat': '42'} data.status = 'OK' data.save() self.assertEqual(second.name, 'User\' data name') self.assertTrue(second.named_by_user) manager.communicate(verbosity=0) second = Data.objects.get(id=second.id) self.assertEqual(second.name, 'User\' data name') self.assertTrue(second.named_by_user)
def create(self, request, *args, **kwargs): """Create a resource.""" collections = request.data.get('collections', []) # check that user has permissions on all collections that Data # object will be added to for collection_id in collections: try: collection = Collection.objects.get(pk=collection_id) except Collection.DoesNotExist: return Response( { 'collections': [ 'Invalid pk "{}" - object does not exist.'.format( collection_id) ] }, status=status.HTTP_400_BAD_REQUEST) if not request.user.has_perm('add_collection', obj=collection): if request.user.has_perm('view_collection', obj=collection): raise exceptions.PermissionDenied( "You don't have `ADD` permission on collection (id: {})." .format(collection_id)) else: raise exceptions.NotFound( "Collection not found (id: {}).".format(collection_id)) # translate processe's slug to id process_slug = request.data.get('process', None) process_query = Process.objects.filter(slug=process_slug) process_query = get_objects_for_user(request.user, 'view_process', process_query) try: process = process_query.latest() except Process.DoesNotExist: return Response( { 'process': [ 'Invalid process slug "{}" - object does not exist.'. format(process_slug) ] }, status=status.HTTP_400_BAD_REQUEST) request.data['process'] = process.pk # perform "get_or_create" if requested - return existing object # if found if kwargs.pop('get_or_create', False): process_input = request.data.get('input', {}) # use default values if they are not given for field_schema, fields, path in iterate_schema( process_input, process.input_schema): if 'default' in field_schema and field_schema[ 'name'] not in fields: dict_dot(process_input, path, field_schema['default']) checksum = get_data_checksum(process_input, process.slug, process.version) data_qs = Data.objects.filter( checksum=checksum, process__persistence__in=[ Process.PERSISTENCE_CACHED, Process.PERSISTENCE_TEMP ], ) data_qs = get_objects_for_user(request.user, 'view_data', data_qs) if data_qs.exists(): data = data_qs.order_by('created').last() serializer = self.get_serializer(data) return Response(serializer.data) # create the objects resp = super(DataViewSet, self).create(request, *args, **kwargs) # run manager manager.communicate() return resp
def test_templatetags(self): input_process = Process.objects.create(name='Input process', contributor=self.contributor, type='data:test:inputobject:', output_schema=[ { 'name': 'test_file', 'type': 'basic:file:' }, ], run={ 'language': 'bash', 'program': """ mkdir -p path/to touch path/to/file.txt re-save-file test_file path/to/file.txt """ }) input_data = Data.objects.create( name='Input Data object', contributor=self.contributor, process=input_process, ) manager.communicate(verbosity=0) process = Process.objects.create( name='Test template tags', requirements={'expression-engine': 'jinja'}, contributor=self.contributor, type='test:data:templatetags:', input_schema=[ { 'name': 'input_data', 'type': 'data:test:inputobject:' }, ], output_schema=[ { 'name': 'name', 'type': 'basic:string:' }, { 'name': 'id', 'type': 'basic:integer:' }, { 'name': 'type', 'type': 'basic:string:' }, { 'name': 'basename', 'type': 'basic:string:' }, { 'name': 'subtype', 'type': 'basic:string:' }, { 'name': 'yesno', 'type': 'basic:string:' }, { 'name': 'datalookup', 'type': 'basic:integer:' }, { 'name': 'file_url', 'type': 'basic:string:' }, ], run={ 'language': 'bash', 'program': """ re-save name "{{ input_data | name }}" re-save id {{ input_data | id }} re-save type {{ input_data | type }} re-save basename "{{ '/foo/bar/moo' | basename }}" re-save subtype "{{ 'data:test:inputobject:' | subtype('data:') }}" re-save yesno "{{ true | yesno('yes', 'no') }}" re-save datalookup "{{ 'input-data-object' | data_by_slug }}" re-save file_url "{{ input_data.test_file | get_url }}" """ }) data = Data.objects.create( name='Data object', contributor=self.contributor, process=process, input={'input_data': input_data.pk}, ) manager.communicate(verbosity=0) data.refresh_from_db() self.assertEqual(data.output['name'], input_data.name) self.assertEqual(data.output['id'], input_data.pk) self.assertEqual(data.output['type'], input_process.type) self.assertEqual(data.output['basename'], 'moo') self.assertEqual(data.output['subtype'], 'True') self.assertEqual(data.output['yesno'], 'yes') self.assertEqual(data.output['datalookup'], input_data.pk) self.assertEqual( data.output['file_url'], 'localhost/data/{}/path/to/file.txt'.format(input_data.pk))