def create_event(cls, event: dict): """Create the event database model.""" event_obj, errors = EventSchema(check_existing=True).load(event) if errors: raise MarshmallowValidationError(errors) # Validate the entries in the payload for payload in event['Payload']: errors = RelationshipSchema(check_existing=True).validate(payload) if errors: raise MarshmallowValidationError(errors) db.session.add(event_obj) return event_obj
def handle_event(cls, event: dict, no_index=False, user_id=None, delayed=True): """Handle an event payload.""" # Raises JSONSchema ValidationError jsonschema.validate(event, EVENT_SCHEMA) # Validate the entries in the payload for payload in event: errors = RelationshipSchema(check_existing=True).validate(payload) if errors: raise MarshmallowValidationError(errors) event_obj = Event(payload=event, status=EventStatus.New, user_id=user_id) db.session.add(event_obj) db.session.commit() event_uuid = str(event_obj.id) idx_enabled = current_app.config['ASCLEPIAS_SEARCH_INDEXING_ENABLED'] \ and (not no_index) if delayed: process_event.delay(event_uuid, indexing_enabled=idx_enabled) else: process_event.apply(kwargs=dict(event_uuid=event_uuid, indexing_enabled=idx_enabled))
def process_event(event_uuid: str, delete=False): """Process an event's payloads.""" # TODO: Should we detect and skip duplicated events? event = Event.get(event_uuid) # TODO: event.payload contains the whole event, not just payload - refactor groups_ids = [] with db.session.begin_nested(): for payload_idx, payload in enumerate(event.payload['Payload']): # TODO: marshmallow validation of all payloads # should be done on first event ingestion (check) relationship, errors = \ RelationshipSchema(check_existing=True).load(payload) # Errors should never happen as the payload is validated # with RelationshipSchema on the event ingestion if errors: raise MarshmallowValidationError(errors) if relationship.id: relationship.deleted = delete db.session.add(relationship) # We need ORM relationship with IDs, since Event has # 'weak' (non-FK) relations to the objects, hence we need # to know the ID upfront relationship = relationship.fetch_or_create_id() create_relation_object_events(event, relationship, payload_idx) id_groups, version_groups = update_groups(relationship) update_metadata(relationship, payload) groups_ids.append( [str(g.id) if g else g for g in id_groups + version_groups]) db.session.commit() for ids in groups_ids: update_indices(*ids)
def handle_event(cls, event: dict, no_index: bool = False, user_id: int = None, eager: bool = False) -> Event: """Handle an event payload.""" # Raises JSONSchema ValidationError jsonschema.validate(event, EVENT_SCHEMA) # Validate the entries in the payload for payload in event: errors = RelationshipSchema(check_existing=True).validate(payload) if errors: raise MarshmallowValidationError(errors) event_obj = Event(payload=event, status=EventStatus.New, user_id=user_id) db.session.add(event_obj) db.session.commit() event_uuid = str(event_obj.id) idx_enabled = current_app.config['ASCLEPIAS_SEARCH_INDEXING_ENABLED'] \ and (not no_index) task = process_event.s(event_uuid=event_uuid, indexing_enabled=idx_enabled) if eager: task.apply(throw=True) else: task.apply_async() return event_obj
def validate_payload(cls, event): """Validate the event payload.""" # TODO: Use invenio-jsonschemas/jsonresolver instead of this # Validate against Event JSONSchema # NOTE: raises `jsonschemas.ValidationError` cls._jsonschema_validator.validate(event) # Validate using marshmallow loader for payload in event: errors = RelationshipSchema(check_existing=True).validate(payload) if errors: raise MarshmallowValidationError( str(errors) + "payload" + str(payload))
def process_event(event_uuid: str, indexing_enabled: bool = True): """Process the event.""" # TODO: Should we detect and skip duplicated events? _set_event_status(event_uuid, EventStatus.Processing) try: event = Event.get(event_uuid) groups_ids = [] with db.session.begin_nested(): for payload_idx, payload in enumerate(event.payload): # TODO: marshmallow validation of all payloads # should be done on first event ingestion (check) relationship, errors = \ RelationshipSchema(check_existing=True).load(payload) # Errors should never happen as the payload is validated # with RelationshipSchema on the event ingestion if errors: raise MarshmallowValidationError(errors) # Skip already known relationships # NOTE: This skips any extra metadata! if relationship.id: continue db.session.add(relationship) # We need ORM relationship with IDs, since Event has # 'weak' (non-FK) relations to the objects, hence we need # to know the ID upfront relationship = relationship.fetch_or_create_id() create_relation_object_events(event, relationship, payload_idx) id_groups, ver_groups = update_groups(relationship) update_metadata_from_event(relationship, payload) groups_ids.append( [str(g.id) if g else g for g in id_groups + ver_groups]) db.session.commit() if indexing_enabled: compacted = compact_indexing_groups(groups_ids) update_indices(*compacted) _set_event_status(event_uuid, EventStatus.Done) event_processed.send(current_app._get_current_object(), event=event) except Exception as exc: _set_event_status(event_uuid, EventStatus.Error) process_event.retry(exc=exc)
def validate_id(self, id): if self.is_create() or int(id) == int(self.instance.id): return raise MarshmallowValidationError('ids do not match')