Example #1
0
    def migrate_process(self, process):
        new = Process()
        new.name = process["label"]
        new.slug = self.process_slug(process["name"])
        new.version = process["version"]
        new.type = process["type"]
        new.description = process.get("description", "")
        new.contributor = self.get_contributor(process["author_id"])
        new.category = process.get("category", "")
        # XXX: Django will change this on create
        new.created = process["date_created"]
        # XXX: Django will change this on save
        new.modified = process["date_modified"]
        new.output_schema = process["output_schema"]
        new.input_schema = process.get("input_schema", {})
        new.persistence = self.persistence_dict[process["persistence"]]
        new.run["script"] = process["run"]["bash"]
        new.save()

        self.migrate_permissions(new, process)

        self.id_mapping["process"][str(process["_id"])] = new.pk
Example #2
0
    def migrate_process(self, process):
        """Migrate processes."""
        new = Process()
        new.name = process[u'label']
        new.slug = self.process_slug(process[u'name'])
        new.version = process[u'version']
        new.type = process[u'type']
        new.description = process.get(u'description', '')
        new.contributor = self.get_contributor(process['author_id'])
        new.category = process.get(u'category', '')
        # XXX: Django will change this on create
        new.created = process[u'date_created']
        # XXX: Django will change this on save
        new.modified = process[u'date_modified']
        new.output_schema = process[u'output_schema']
        new.input_schema = process.get(u'input_schema', {})
        new.persistence = self.persistence_dict[process[u'persistence']]
        new.run['script'] = process[u'run'][u'bash']  # pylint: disable=unsubscriptable-object
        new.save()

        self.migrate_permissions(new, process)

        self.id_mapping['process'][str(process[u'_id'])] = new.pk
Example #3
0
    def migrate_process(self, process):
        """Migrate processes."""
        new = Process()
        new.name = process[u'label']
        new.slug = self.process_slug(process[u'name'])
        new.version = process[u'version']
        new.type = process[u'type']
        new.description = process.get(u'description', '')
        new.contributor = self.get_contributor(process['author_id'])
        new.category = process.get(u'category', '')
        # XXX: Django will change this on create
        new.created = process[u'date_created']
        # XXX: Django will change this on save
        new.modified = process[u'date_modified']
        new.output_schema = process[u'output_schema']
        new.input_schema = process.get(u'input_schema', {})
        new.persistence = self.persistence_dict[process[u'persistence']]
        new.run['script'] = process[u'run'][u'bash']  # pylint: disable=unsubscriptable-object
        new.save()

        self.migrate_permissions(new, process)

        self.id_mapping['process'][str(process[u'_id'])] = new.pk
Example #4
0
    def migrate_data(self, data):
        contributor = self.get_contributor(data["author_id"])

        # DESCRIPTOR SCHEMA ############################################
        ds_fields = []
        ds_fields.extend(data.get("static_schema", []))
        ds_fields.extend(data.get("var_template", []))
        ds_fields.sort(key=lambda d: d["name"])
        ds_fields_dumped = json.dumps(ds_fields)

        if ds_fields_dumped in self.descriptor_schema_index:
            descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
        else:
            descriptor_schema = DescriptorSchema(schema=ds_fields)
            descriptor_schema.name = "data_{}_descriptor".format(data["_id"])
            descriptor_schema.contributor = contributor
            descriptor_schema.save()

            self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema

        descriptor = {}
        descriptor.update(data.get("static", {}))
        descriptor.update(data.get("var", {}))

        # PROCESS ######################################################
        if "processor_version" not in data:
            data["processor_version"] = "0.0.0"

        process_slug = self.process_slug(data["processor_name"])
        process_version = data["processor_version"]
        try:
            process = Process.objects.get(slug=process_slug, version=process_version)
        except Process.DoesNotExist:
            latest = Process.objects.filter(slug=process_slug).order_by("-version").first()

            if latest:
                process = Process()
                process.name = latest.name
                process.slug = latest.slug
                process.category = latest.category
                process.description = latest.description
                process.contributor = latest.contributor

                process.version = process_version
                process.type = data["type"]
                process.output_schema = data["output_schema"]
                process.input_schema = data.get("input_schema", {})
                process.persistence = self.persistence_dict[data["persistence"]]

                process.run["script"] = 'gen-require common\ngen-error "Depricated process, use the latest version."'

                # XXX
                # process.created =
                # process.modified =

                process.save()

                # copy permissions from latest process
                for user, perms in get_users_with_perms(latest, attach_perms=True).iteritems():
                    for perm in perms:
                        assign_perm(perm, user, process)
                for group, perms in get_groups_with_perms(latest, attach_perms=True).iteritems():
                    for perm in perms:
                        assign_perm(perm, group, process)
            else:
                # Create dummy processor if there is no other version
                dummy_name = "Dummy processor of type {}".format(data["type"])
                try:
                    process = Process.objects.get(name=dummy_name)
                except Process.DoesNotExist:
                    process = Process.objects.create(
                        name=dummy_name,
                        slug="non-existent",
                        contributor=get_user_model().objects.filter(is_superuser=True).first(),
                        type=data["type"],
                        category="data:non-existent",
                        run={"script": {'gen-require common\ngen-error "This processor is not intendent to be run."'}},
                    )

        # DATA #########################################################
        new = Data()
        new.name = data.get("static", {}).get("name", "")
        if len(new.name) > 100:
            self.long_names.append(new.name)
            new.name = new.name[:97] + "..."
        new.status = self.status_dict[data["status"]]
        new.process = process
        new.contributor = contributor
        new.input = data["input"] if "input" in data else {}
        new.output = data["output"]
        new.descriptor_schema = descriptor_schema
        new.descriptor = descriptor
        new.checksum = data.get("checksum", "")
        # XXX: Django will change this on create
        new.created = data["date_created"]
        # XXX: Django will change this on save
        new.modified = data["date_modified"]
        if "date_start" in data and "date_finish" in data:
            new.started = data["date_start"]
            new.finished = data["date_finish"]
        elif "date_finish" in data:
            new.started = data["date_finish"]
            new.finished = data["date_finish"]
        elif "date_start" in data:
            new.started = data["date_start"]
            new.finished = data["date_start"]
        else:
            new.started = datetime.fromtimestamp(0)
            new.finished = datetime.fromtimestamp(0)
        new.save()

        for case_id in data["case_ids"]:
            try:
                collection = Collection.objects.get(pk=self.id_mapping["collection"][str(case_id)])
            except KeyError:
                self.missing_collections.add(str(case_id))
                continue
            collection.data.add(new)

        for field_schema, fields, path in iterate_fields(data["output"], data["output_schema"], ""):
            if "type" in field_schema and field_schema["type"].startswith("basic:json:"):
                self.storage_index[fields[field_schema["name"]]] = {"id": new.pk, "path": path}

        self.migrate_permissions(new, data)

        self.id_mapping["data"][str(data["_id"])] = new.pk

        # DESCRIPTOR SCHEMA PERMISSIONS ################################
        for user in get_users_with_perms(new):
            assign_perm("view_descriptorschema", user, obj=descriptor_schema)

        for group in get_groups_with_perms(new):
            assign_perm("view_descriptorschema", group, obj=descriptor_schema)
Example #5
0
    def migrate_data(self, data):
        """Migrate data."""
        contributor = self.get_contributor(data[u'author_id'])

        # DESCRIPTOR SCHEMA ############################################
        ds_fields = []
        ds_fields.extend(data.get(u'static_schema', []))
        ds_fields.extend(data.get(u'var_template', []))
        ds_fields.sort(key=lambda d: d[u'name'])
        ds_fields_dumped = json.dumps(ds_fields)

        if ds_fields_dumped in self.descriptor_schema_index:
            descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
        else:
            descriptor_schema = DescriptorSchema(schema=ds_fields)
            descriptor_schema.name = 'data_{}_descriptor'.format(data[u'_id'])
            descriptor_schema.contributor = contributor
            descriptor_schema.save()

            self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema

        descriptor = {}
        descriptor.update(data.get(u'static', {}))
        descriptor.update(data.get(u'var', {}))

        # PROCESS ######################################################
        if u'processor_version' not in data:
            data[u'processor_version'] = '0.0.0'

        process_slug = self.process_slug(data[u'processor_name'])
        process_version = data[u'processor_version']
        try:
            process = Process.objects.get(slug=process_slug, version=process_version)
        except Process.DoesNotExist:
            latest = Process.objects.filter(slug=process_slug).order_by('-version').first()

            if latest:
                process = Process()
                process.name = latest.name
                process.slug = latest.slug
                process.category = latest.category
                process.description = latest.description
                process.contributor = latest.contributor

                process.version = process_version
                process.type = data[u'type']
                process.output_schema = data[u'output_schema']
                process.input_schema = data.get(u'input_schema', {})
                process.persistence = self.persistence_dict[data[u'persistence']]

                process.run['script'] = 'gen-require common\ngen-error "Depricated process, use the latest version."'  # noqa pylint: disable=unsubscriptable-object

                # XXX
                # process.created =
                # process.modified =

                process.save()

                # copy permissions from latest process
                for user, perms in six.iteritems(get_users_with_perms(latest, attach_perms=True)):
                    for perm in perms:
                        assign_perm(perm, user, process)
                for group, perms in six.iteritems(get_groups_with_perms(latest, attach_perms=True)):
                    for perm in perms:
                        assign_perm(perm, group, process)
            else:
                # Create dummy processor if there is no other version
                dummy_name = 'Dummy processor of type {}'.format(data[u'type'])
                try:
                    process = Process.objects.get(name=dummy_name)
                except Process.DoesNotExist:
                    process = Process.objects.create(
                        name=dummy_name,
                        slug='non-existent',
                        contributor=get_user_model().objects.filter(is_superuser=True).first(),
                        type=data[u'type'],
                        category='data:non-existent',
                        run={'script': {'gen-require common\ngen-error "This processor is not intendent to be run."'}},
                    )

        # DATA #########################################################
        new = Data()
        new.name = data.get(u'static', {}).get(u'name', '')
        if len(new.name) > 100:
            self.long_names.append(new.name)
            new.name = new.name[:97] + '...'
        new.status = self.status_dict[data[u'status']]
        new.process = process
        new.contributor = contributor
        new.input = data[u'input'] if u'input' in data else {}
        new.output = data[u'output']
        new.descriptor_schema = descriptor_schema
        new.descriptor = descriptor
        new.checksum = data.get(u'checksum', '')
        # XXX: Django will change this on create
        new.created = data[u'date_created']
        # XXX: Django will change this on save
        new.modified = data[u'date_modified']
        if u'date_start' in data and u'date_finish' in data:
            new.started = data[u'date_start']
            new.finished = data[u'date_finish']
        elif u'date_finish' in data:
            new.started = data[u'date_finish']
            new.finished = data[u'date_finish']
        elif u'date_start' in data:
            new.started = data[u'date_start']
            new.finished = data[u'date_start']
        else:
            new.started = datetime.fromtimestamp(0)
            new.finished = datetime.fromtimestamp(0)
        new.save()

        for case_id in data[u'case_ids']:
            try:
                collection = Collection.objects.get(pk=self.id_mapping[u'collection'][str(case_id)])
            except KeyError:
                self.missing_collections.add(str(case_id))
                continue
            collection.data.add(new)

        for field_schema, fields, path in iterate_fields(data[u'output'], data[u'output_schema'], ''):
            if 'type' in field_schema and field_schema['type'].startswith('basic:json:'):
                self.storage_index[fields[field_schema['name']]] = {
                    'id': new.pk,
                    'path': path,
                }

        self.migrate_permissions(new, data)

        self.id_mapping['data'][str(data[u'_id'])] = new.pk

        # DESCRIPTOR SCHEMA PERMISSIONS ################################
        for user in get_users_with_perms(new):
            assign_perm('view_descriptorschema', user, obj=descriptor_schema)

        for group in get_groups_with_perms(new):
            assign_perm('view_descriptorschema', group, obj=descriptor_schema)
Example #6
0
    def migrate_data(self, data):
        """Migrate data."""
        contributor = self.get_contributor(data[u'author_id'])

        # DESCRIPTOR SCHEMA ############################################
        ds_fields = []
        ds_fields.extend(data.get(u'static_schema', []))
        ds_fields.extend(data.get(u'var_template', []))
        ds_fields.sort(key=lambda d: d[u'name'])
        ds_fields_dumped = json.dumps(ds_fields)

        if ds_fields_dumped in self.descriptor_schema_index:
            descriptor_schema = self.descriptor_schema_index[ds_fields_dumped]
        else:
            descriptor_schema = DescriptorSchema(schema=ds_fields)
            descriptor_schema.name = 'data_{}_descriptor'.format(data[u'_id'])
            descriptor_schema.contributor = contributor
            descriptor_schema.save()

            self.descriptor_schema_index[ds_fields_dumped] = descriptor_schema

        descriptor = {}
        descriptor.update(data.get(u'static', {}))
        descriptor.update(data.get(u'var', {}))

        # PROCESS ######################################################
        if u'processor_version' not in data:
            data[u'processor_version'] = '0.0.0'

        process_slug = self.process_slug(data[u'processor_name'])
        process_version = data[u'processor_version']
        try:
            process = Process.objects.get(slug=process_slug,
                                          version=process_version)
        except Process.DoesNotExist:
            latest = Process.objects.filter(
                slug=process_slug).order_by('-version').first()

            if latest:
                process = Process()
                process.name = latest.name
                process.slug = latest.slug
                process.category = latest.category
                process.description = latest.description
                process.contributor = latest.contributor

                process.version = process_version
                process.type = data[u'type']
                process.output_schema = data[u'output_schema']
                process.input_schema = data.get(u'input_schema', {})
                process.persistence = self.persistence_dict[
                    data[u'persistence']]

                process.run[
                    'script'] = 'gen-require common\ngen-error "Depricated process, use the latest version."'  # noqa pylint: disable=unsubscriptable-object

                # XXX
                # process.created =
                # process.modified =

                process.save()

                # copy permissions from latest process
                for user, perms in six.iteritems(
                        get_users_with_perms(latest, attach_perms=True)):
                    for perm in perms:
                        assign_perm(perm, user, process)
                for group, perms in six.iteritems(
                        get_groups_with_perms(latest, attach_perms=True)):
                    for perm in perms:
                        assign_perm(perm, group, process)
            else:
                # Create dummy processor if there is no other version
                dummy_name = 'Dummy processor of type {}'.format(data[u'type'])
                try:
                    process = Process.objects.get(name=dummy_name)
                except Process.DoesNotExist:
                    process = Process.objects.create(
                        name=dummy_name,
                        slug='non-existent',
                        contributor=get_user_model().objects.filter(
                            is_superuser=True).first(),
                        type=data[u'type'],
                        category='data:non-existent',
                        run={
                            'script': {
                                'gen-require common\ngen-error "This processor is not intendent to be run."'
                            }
                        },
                    )

        # DATA #########################################################
        new = Data()
        new.name = data.get(u'static', {}).get(u'name', '')
        if len(new.name) > 100:
            self.long_names.append(new.name)
            new.name = new.name[:97] + '...'
        new.status = self.status_dict[data[u'status']]
        new.process = process
        new.contributor = contributor
        new.input = data[u'input'] if u'input' in data else {}
        new.output = data[u'output']
        new.descriptor_schema = descriptor_schema
        new.descriptor = descriptor
        new.checksum = data.get(u'checksum', '')
        # XXX: Django will change this on create
        new.created = data[u'date_created']
        # XXX: Django will change this on save
        new.modified = data[u'date_modified']
        if u'date_start' in data and u'date_finish' in data:
            new.started = data[u'date_start']
            new.finished = data[u'date_finish']
        elif u'date_finish' in data:
            new.started = data[u'date_finish']
            new.finished = data[u'date_finish']
        elif u'date_start' in data:
            new.started = data[u'date_start']
            new.finished = data[u'date_start']
        else:
            new.started = datetime.fromtimestamp(0)
            new.finished = datetime.fromtimestamp(0)
        new.save()

        for case_id in data[u'case_ids']:
            try:
                collection = Collection.objects.get(
                    pk=self.id_mapping[u'collection'][str(case_id)])
            except KeyError:
                self.missing_collections.add(str(case_id))
                continue
            collection.data.add(new)

        for field_schema, fields, path in iterate_fields(
                data[u'output'], data[u'output_schema'], ''):
            if 'type' in field_schema and field_schema['type'].startswith(
                    'basic:json:'):
                self.storage_index[fields[field_schema['name']]] = {
                    'id': new.pk,
                    'path': path,
                }

        self.migrate_permissions(new, data)

        self.id_mapping['data'][str(data[u'_id'])] = new.pk

        # DESCRIPTOR SCHEMA PERMISSIONS ################################
        for user in get_users_with_perms(new):
            assign_perm('view_descriptorschema', user, obj=descriptor_schema)

        for group in get_groups_with_perms(new):
            assign_perm('view_descriptorschema', group, obj=descriptor_schema)