Ejemplo n.º 1
0
    def setUp(self):
        super(BackendTest, self).setUp()

        self.p = Process(slug='test-processor',
                         name='Test Process',
                         contributor=self.contributor,
                         type='data:test',
                         version=1)
        self.p.save()

        self.d = Data(slug='test-data',
                      name='Test Data',
                      contributor=self.contributor,
                      process=self.p)
        self.d.save()
Ejemplo n.º 2
0
class BackendTest(TestCase):
    def setUp(self):
        u = get_user_model().objects.create_superuser('test', '*****@*****.**', 'test')
        self.p = Process(slug='test-processor',
                         name='Test Process',
                         contributor=u,
                         type='data:test',
                         version=1)
        self.p.save()

        self.d = Data(slug='test-data',
                      name='Test Data',
                      contributor=u,
                      process=self.p)
        self.d.save()

    def tearDown(self):
        for data in Data.objects.all():
            data_dir = os.path.join(settings.FLOW_EXECUTOR['DATA_DIR'], str(data.id))
            shutil.rmtree(data_dir, ignore_errors=True)

    def test_manager(self):
        manager.communicate(verbosity=0)

    def test_dtlbash(self):
        self.p.slug = 'test-processor-dtlbash'
        self.p.run = {'script': """
gen-info \"Test processor info\"
gen-warning \"Test processor warning\"

echo '{"proc.info": "foo"}'
"""}
        self.p.save()

        self.d.slug = 'test-data-dtlbash'
        self.d.process = self.p
        self.d.save()
        self.d = Data(id=self.d.id)
Ejemplo n.º 3
0
    def setUp(self):
        super().setUp()

        self.process = Process(
            output_schema=[
                {'name': 'test_file', 'type': 'basic:file:', 'required': False},
                {'name': 'file_list', 'type': 'list:basic:file:', 'required': False}
            ]
        )
        self.data = Data(
            pk=13,
            process=self.process,
            output={'test_file': {'file': 'test_file.tmp'}}
        )
Ejemplo n.º 4
0
    def setUp(self):
        u = get_user_model().objects.create_superuser('test', '*****@*****.**', 'test')
        self.p = Process(slug='test-processor',
                         name='Test Process',
                         contributor=u,
                         type='data:test',
                         version=1)
        self.p.save()

        self.d = Data(slug='test-data',
                      name='Test Data',
                      contributor=u,
                      process=self.p)
        self.d.save()
Ejemplo n.º 5
0
class BackendTest(TestCase):
    def setUp(self):
        super(BackendTest, self).setUp()

        self.p = Process(slug='test-processor',
                         name='Test Process',
                         contributor=self.contributor,
                         type='data:test',
                         version=1)
        self.p.save()

        self.d = Data(slug='test-data',
                      name='Test Data',
                      contributor=self.contributor,
                      process=self.p)
        self.d.save()

    def test_manager(self):
        manager.communicate(verbosity=0)

    def test_dtlbash(self):
        self.p.slug = 'test-processor-dtlbash'
        self.p.run = {
            'script':
            """
gen-info \"Test processor info\"
gen-warning \"Test processor warning\"

echo '{"proc.info": "foo"}'
"""
        }
        self.p.save()

        self.d.slug = 'test-data-dtlbash'
        self.d.process = self.p
        self.d.save()
        self.d = Data(id=self.d.id)
Ejemplo n.º 6
0
    def test_checksum_consistency(self):
        process = Process(version="1.0.0", slug="my-process")
        data = Data()

        data.input = {"tss": 0, "genome": "HG19"}
        checksum = get_data_checksum(data.input, process.slug, process.version)
        self.assertEqual(
            checksum,
            "ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c")

        data.input = {"genome": "HG19", "tss": 0}
        checksum = get_data_checksum(data.input, process.slug, process.version)
        self.assertEqual(
            checksum,
            "ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c")
Ejemplo n.º 7
0
    def test_checksum_consistency(self):
        process = Process(version='1.0.0', slug='my-process')
        data = Data()

        data.input = {'tss': 0, 'genome': 'HG19'}
        checksum = get_data_checksum(data.input, process.slug, process.version)
        self.assertEqual(
            checksum,
            'ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c')

        data.input = {'genome': 'HG19', 'tss': 0}
        checksum = get_data_checksum(data.input, process.slug, process.version)
        self.assertEqual(
            checksum,
            'ca322c2bb48b58eea3946e624fe6cfdc53c2cc12478465b6f0ca2d722e280c4c')
Ejemplo n.º 8
0
    def migrate_data(self, data):
        contributor = self.get_contributor(data["author_id"])

        # DESCRIPTOR SCHEMA ############################################
        ds_fields = []
        ds_fields.extend(data.get("static_schema", []))
        ds_fields.extend(data.get("var_template", []))
        ds_fields.sort(key=lambda d: d["name"])
        ds_fields_dumped = json.dumps(ds_fields)

        if ds_fields_dumped in self.descriptor_schema_index:
            descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
        else:
            descriptor_schema = DescriptorSchema(schema=ds_fields)
            descriptor_schema.name = "data_{}_descriptor".format(data["_id"])
            descriptor_schema.contributor = contributor
            descriptor_schema.save()

            self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema

        descriptor = {}
        descriptor.update(data.get("static", {}))
        descriptor.update(data.get("var", {}))

        # PROCESS ######################################################
        if "processor_version" not in data:
            data["processor_version"] = "0.0.0"

        process_slug = self.process_slug(data["processor_name"])
        process_version = data["processor_version"]
        try:
            process = Process.objects.get(slug=process_slug, version=process_version)
        except Process.DoesNotExist:
            latest = Process.objects.filter(slug=process_slug).order_by("-version").first()

            if latest:
                process = Process()
                process.name = latest.name
                process.slug = latest.slug
                process.category = latest.category
                process.description = latest.description
                process.contributor = latest.contributor

                process.version = process_version
                process.type = data["type"]
                process.output_schema = data["output_schema"]
                process.input_schema = data.get("input_schema", {})
                process.persistence = self.persistence_dict[data["persistence"]]

                process.run["script"] = 'gen-require common\ngen-error "Depricated process, use the latest version."'

                # XXX
                # process.created =
                # process.modified =

                process.save()

                # copy permissions from latest process
                for user, perms in get_users_with_perms(latest, attach_perms=True).iteritems():
                    for perm in perms:
                        assign_perm(perm, user, process)
                for group, perms in get_groups_with_perms(latest, attach_perms=True).iteritems():
                    for perm in perms:
                        assign_perm(perm, group, process)
            else:
                # Create dummy processor if there is no other version
                dummy_name = "Dummy processor of type {}".format(data["type"])
                try:
                    process = Process.objects.get(name=dummy_name)
                except Process.DoesNotExist:
                    process = Process.objects.create(
                        name=dummy_name,
                        slug="non-existent",
                        contributor=get_user_model().objects.filter(is_superuser=True).first(),
                        type=data["type"],
                        category="data:non-existent",
                        run={"script": {'gen-require common\ngen-error "This processor is not intendent to be run."'}},
                    )

        # DATA #########################################################
        new = Data()
        new.name = data.get("static", {}).get("name", "")
        if len(new.name) > 100:
            self.long_names.append(new.name)
            new.name = new.name[:97] + "..."
        new.status = self.status_dict[data["status"]]
        new.process = process
        new.contributor = contributor
        new.input = data["input"] if "input" in data else {}
        new.output = data["output"]
        new.descriptor_schema = descriptor_schema
        new.descriptor = descriptor
        new.checksum = data.get("checksum", "")
        # XXX: Django will change this on create
        new.created = data["date_created"]
        # XXX: Django will change this on save
        new.modified = data["date_modified"]
        if "date_start" in data and "date_finish" in data:
            new.started = data["date_start"]
            new.finished = data["date_finish"]
        elif "date_finish" in data:
            new.started = data["date_finish"]
            new.finished = data["date_finish"]
        elif "date_start" in data:
            new.started = data["date_start"]
            new.finished = data["date_start"]
        else:
            new.started = datetime.fromtimestamp(0)
            new.finished = datetime.fromtimestamp(0)
        new.save()

        for case_id in data["case_ids"]:
            try:
                collection = Collection.objects.get(pk=self.id_mapping["collection"][str(case_id)])
            except KeyError:
                self.missing_collections.add(str(case_id))
                continue
            collection.data.add(new)

        for field_schema, fields, path in iterate_fields(data["output"], data["output_schema"], ""):
            if "type" in field_schema and field_schema["type"].startswith("basic:json:"):
                self.storage_index[fields[field_schema["name"]]] = {"id": new.pk, "path": path}

        self.migrate_permissions(new, data)

        self.id_mapping["data"][str(data["_id"])] = new.pk

        # DESCRIPTOR SCHEMA PERMISSIONS ################################
        for user in get_users_with_perms(new):
            assign_perm("view_descriptorschema", user, obj=descriptor_schema)

        for group in get_groups_with_perms(new):
            assign_perm("view_descriptorschema", group, obj=descriptor_schema)
Ejemplo n.º 9
0
    def migrate_process(self, process):
        new = Process()
        new.name = process["label"]
        new.slug = self.process_slug(process["name"])
        new.version = process["version"]
        new.type = process["type"]
        new.description = process.get("description", "")
        new.contributor = self.get_contributor(process["author_id"])
        new.category = process.get("category", "")
        # XXX: Django will change this on create
        new.created = process["date_created"]
        # XXX: Django will change this on save
        new.modified = process["date_modified"]
        new.output_schema = process["output_schema"]
        new.input_schema = process.get("input_schema", {})
        new.persistence = self.persistence_dict[process["persistence"]]
        new.run["script"] = process["run"]["bash"]
        new.save()

        self.migrate_permissions(new, process)

        self.id_mapping["process"][str(process["_id"])] = new.pk
Ejemplo n.º 10
0
    def migrate_data(self, data):
        """Migrate data."""
        contributor = self.get_contributor(data[u'author_id'])

        # DESCRIPTOR SCHEMA ############################################
        ds_fields = []
        ds_fields.extend(data.get(u'static_schema', []))
        ds_fields.extend(data.get(u'var_template', []))
        ds_fields.sort(key=lambda d: d[u'name'])
        ds_fields_dumped = json.dumps(ds_fields)

        if ds_fields_dumped in self.descriptor_schema_index:
            descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
        else:
            descriptor_schema = DescriptorSchema(schema=ds_fields)
            descriptor_schema.name = 'data_{}_descriptor'.format(data[u'_id'])
            descriptor_schema.contributor = contributor
            descriptor_schema.save()

            self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema

        descriptor = {}
        descriptor.update(data.get(u'static', {}))
        descriptor.update(data.get(u'var', {}))

        # PROCESS ######################################################
        if u'processor_version' not in data:
            data[u'processor_version'] = '0.0.0'

        process_slug = self.process_slug(data[u'processor_name'])
        process_version = data[u'processor_version']
        try:
            process = Process.objects.get(slug=process_slug, version=process_version)
        except Process.DoesNotExist:
            latest = Process.objects.filter(slug=process_slug).order_by('-version').first()

            if latest:
                process = Process()
                process.name = latest.name
                process.slug = latest.slug
                process.category = latest.category
                process.description = latest.description
                process.contributor = latest.contributor

                process.version = process_version
                process.type = data[u'type']
                process.output_schema = data[u'output_schema']
                process.input_schema = data.get(u'input_schema', {})
                process.persistence = self.persistence_dict[data[u'persistence']]

                process.run['script'] = 'gen-require common\ngen-error "Depricated process, use the latest version."'  # noqa pylint: disable=unsubscriptable-object

                # XXX
                # process.created =
                # process.modified =

                process.save()

                # copy permissions from latest process
                for user, perms in six.iteritems(get_users_with_perms(latest, attach_perms=True)):
                    for perm in perms:
                        assign_perm(perm, user, process)
                for group, perms in six.iteritems(get_groups_with_perms(latest, attach_perms=True)):
                    for perm in perms:
                        assign_perm(perm, group, process)
            else:
                # Create dummy processor if there is no other version
                dummy_name = 'Dummy processor of type {}'.format(data[u'type'])
                try:
                    process = Process.objects.get(name=dummy_name)
                except Process.DoesNotExist:
                    process = Process.objects.create(
                        name=dummy_name,
                        slug='non-existent',
                        contributor=get_user_model().objects.filter(is_superuser=True).first(),
                        type=data[u'type'],
                        category='data:non-existent',
                        run={'script': {'gen-require common\ngen-error "This processor is not intendent to be run."'}},
                    )

        # DATA #########################################################
        new = Data()
        new.name = data.get(u'static', {}).get(u'name', '')
        if len(new.name) > 100:
            self.long_names.append(new.name)
            new.name = new.name[:97] + '...'
        new.status = self.status_dict[data[u'status']]
        new.process = process
        new.contributor = contributor
        new.input = data[u'input'] if u'input' in data else {}
        new.output = data[u'output']
        new.descriptor_schema = descriptor_schema
        new.descriptor = descriptor
        new.checksum = data.get(u'checksum', '')
        # XXX: Django will change this on create
        new.created = data[u'date_created']
        # XXX: Django will change this on save
        new.modified = data[u'date_modified']
        if u'date_start' in data and u'date_finish' in data:
            new.started = data[u'date_start']
            new.finished = data[u'date_finish']
        elif u'date_finish' in data:
            new.started = data[u'date_finish']
            new.finished = data[u'date_finish']
        elif u'date_start' in data:
            new.started = data[u'date_start']
            new.finished = data[u'date_start']
        else:
            new.started = datetime.fromtimestamp(0)
            new.finished = datetime.fromtimestamp(0)
        new.save()

        for case_id in data[u'case_ids']:
            try:
                collection = Collection.objects.get(pk=self.id_mapping[u'collection'][str(case_id)])
            except KeyError:
                self.missing_collections.add(str(case_id))
                continue
            collection.data.add(new)

        for field_schema, fields, path in iterate_fields(data[u'output'], data[u'output_schema'], ''):
            if 'type' in field_schema and field_schema['type'].startswith('basic:json:'):
                self.storage_index[fields[field_schema['name']]] = {
                    'id': new.pk,
                    'path': path,
                }

        self.migrate_permissions(new, data)

        self.id_mapping['data'][str(data[u'_id'])] = new.pk

        # DESCRIPTOR SCHEMA PERMISSIONS ################################
        for user in get_users_with_perms(new):
            assign_perm('view_descriptorschema', user, obj=descriptor_schema)

        for group in get_groups_with_perms(new):
            assign_perm('view_descriptorschema', group, obj=descriptor_schema)
Ejemplo n.º 11
0
    def migrate_process(self, process):
        """Migrate processes."""
        new = Process()
        new.name = process[u'label']
        new.slug = self.process_slug(process[u'name'])
        new.version = process[u'version']
        new.type = process[u'type']
        new.description = process.get(u'description', '')
        new.contributor = self.get_contributor(process['author_id'])
        new.category = process.get(u'category', '')
        # XXX: Django will change this on create
        new.created = process[u'date_created']
        # XXX: Django will change this on save
        new.modified = process[u'date_modified']
        new.output_schema = process[u'output_schema']
        new.input_schema = process.get(u'input_schema', {})
        new.persistence = self.persistence_dict[process[u'persistence']]
        new.run['script'] = process[u'run'][u'bash']  # pylint: disable=unsubscriptable-object
        new.save()

        self.migrate_permissions(new, process)

        self.id_mapping['process'][str(process[u'_id'])] = new.pk
Ejemplo n.º 12
0
    def migrate_data(self, data):
        """Migrate data."""
        contributor = self.get_contributor(data[u'author_id'])

        # DESCRIPTOR SCHEMA ############################################
        ds_fields = []
        ds_fields.extend(data.get(u'static_schema', []))
        ds_fields.extend(data.get(u'var_template', []))
        ds_fields.sort(key=lambda d: d[u'name'])
        ds_fields_dumped = json.dumps(ds_fields)

        if ds_fields_dumped in self.descriptor_schema_index:
            descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
        else:
            descriptor_schema = DescriptorSchema(schema=ds_fields)
            descriptor_schema.name = 'data_{}_descriptor'.format(data[u'_id'])
            descriptor_schema.contributor = contributor
            descriptor_schema.save()

            self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema

        descriptor = {}
        descriptor.update(data.get(u'static', {}))
        descriptor.update(data.get(u'var', {}))

        # PROCESS ######################################################
        if u'processor_version' not in data:
            data[u'processor_version'] = '0.0.0'

        process_slug = self.process_slug(data[u'processor_name'])
        process_version = data[u'processor_version']
        try:
            process = Process.objects.get(slug=process_slug,
                                          version=process_version)
        except Process.DoesNotExist:
            latest = Process.objects.filter(
                slug=process_slug).order_by('-version').first()

            if latest:
                process = Process()
                process.name = latest.name
                process.slug = latest.slug
                process.category = latest.category
                process.description = latest.description
                process.contributor = latest.contributor

                process.version = process_version
                process.type = data[u'type']
                process.output_schema = data[u'output_schema']
                process.input_schema = data.get(u'input_schema', {})
                process.persistence = self.persistence_dict[
                    data[u'persistence']]

                process.run[
                    'script'] = 'gen-require common\ngen-error "Depricated process, use the latest version."'  # noqa pylint: disable=unsubscriptable-object

                # XXX
                # process.created =
                # process.modified =

                process.save()

                # copy permissions from latest process
                for user, perms in six.iteritems(
                        get_users_with_perms(latest, attach_perms=True)):
                    for perm in perms:
                        assign_perm(perm, user, process)
                for group, perms in six.iteritems(
                        get_groups_with_perms(latest, attach_perms=True)):
                    for perm in perms:
                        assign_perm(perm, group, process)
            else:
                # Create dummy processor if there is no other version
                dummy_name = 'Dummy processor of type {}'.format(data[u'type'])
                try:
                    process = Process.objects.get(name=dummy_name)
                except Process.DoesNotExist:
                    process = Process.objects.create(
                        name=dummy_name,
                        slug='non-existent',
                        contributor=get_user_model().objects.filter(
                            is_superuser=True).first(),
                        type=data[u'type'],
                        category='data:non-existent',
                        run={
                            'script': {
                                'gen-require common\ngen-error "This processor is not intendent to be run."'
                            }
                        },
                    )

        # DATA #########################################################
        new = Data()
        new.name = data.get(u'static', {}).get(u'name', '')
        if len(new.name) > 100:
            self.long_names.append(new.name)
            new.name = new.name[:97] + '...'
        new.status = self.status_dict[data[u'status']]
        new.process = process
        new.contributor = contributor
        new.input = data[u'input'] if u'input' in data else {}
        new.output = data[u'output']
        new.descriptor_schema = descriptor_schema
        new.descriptor = descriptor
        new.checksum = data.get(u'checksum', '')
        # XXX: Django will change this on create
        new.created = data[u'date_created']
        # XXX: Django will change this on save
        new.modified = data[u'date_modified']
        if u'date_start' in data and u'date_finish' in data:
            new.started = data[u'date_start']
            new.finished = data[u'date_finish']
        elif u'date_finish' in data:
            new.started = data[u'date_finish']
            new.finished = data[u'date_finish']
        elif u'date_start' in data:
            new.started = data[u'date_start']
            new.finished = data[u'date_start']
        else:
            new.started = datetime.fromtimestamp(0)
            new.finished = datetime.fromtimestamp(0)
        new.save()

        for case_id in data[u'case_ids']:
            try:
                collection = Collection.objects.get(
                    pk=self.id_mapping[u'collection'][str(case_id)])
            except KeyError:
                self.missing_collections.add(str(case_id))
                continue
            collection.data.add(new)

        for field_schema, fields, path in iterate_fields(
                data[u'output'], data[u'output_schema'], ''):
            if 'type' in field_schema and field_schema['type'].startswith(
                    'basic:json:'):
                self.storage_index[fields[field_schema['name']]] = {
                    'id': new.pk,
                    'path': path,
                }

        self.migrate_permissions(new, data)

        self.id_mapping['data'][str(data[u'_id'])] = new.pk

        # DESCRIPTOR SCHEMA PERMISSIONS ################################
        for user in get_users_with_perms(new):
            assign_perm('view_descriptorschema', user, obj=descriptor_schema)

        for group in get_groups_with_perms(new):
            assign_perm('view_descriptorschema', group, obj=descriptor_schema)
Ejemplo n.º 13
0
    def migrate_process(self, process):
        """Migrate processes."""
        new = Process()
        new.name = process[u'label']
        new.slug = self.process_slug(process[u'name'])
        new.version = process[u'version']
        new.type = process[u'type']
        new.description = process.get(u'description', '')
        new.contributor = self.get_contributor(process['author_id'])
        new.category = process.get(u'category', '')
        # XXX: Django will change this on create
        new.created = process[u'date_created']
        # XXX: Django will change this on save
        new.modified = process[u'date_modified']
        new.output_schema = process[u'output_schema']
        new.input_schema = process.get(u'input_schema', {})
        new.persistence = self.persistence_dict[process[u'persistence']]
        new.run['script'] = process[u'run'][u'bash']  # pylint: disable=unsubscriptable-object
        new.save()

        self.migrate_permissions(new, process)

        self.id_mapping['process'][str(process[u'_id'])] = new.pk