def create_event(cls, event: dict):
"""Create the event database model."""
event_obj, errors = EventSchema(check_existing=True).load(event)
if errors:
raise MarshmallowValidationError(errors)
# Validate the entries in the payload
for payload in event['Payload']:
errors = RelationshipSchema(check_existing=True).validate(payload)
if errors:
raise MarshmallowValidationError(errors)
db.session.add(event_obj)
return event_obj
def handle_event(cls,
event: dict,
no_index=False,
user_id=None,
delayed=True):
"""Handle an event payload."""
# Raises JSONSchema ValidationError
jsonschema.validate(event, EVENT_SCHEMA)
# Validate the entries in the payload
for payload in event:
errors = RelationshipSchema(check_existing=True).validate(payload)
if errors:
raise MarshmallowValidationError(errors)
event_obj = Event(payload=event,
status=EventStatus.New,
user_id=user_id)
db.session.add(event_obj)
db.session.commit()
event_uuid = str(event_obj.id)
idx_enabled = current_app.config['ASCLEPIAS_SEARCH_INDEXING_ENABLED'] \
and (not no_index)
if delayed:
process_event.delay(event_uuid, indexing_enabled=idx_enabled)
else:
process_event.apply(kwargs=dict(event_uuid=event_uuid,
indexing_enabled=idx_enabled))
def process_event(event_uuid: str, delete=False):
"""Process an event's payloads."""
# TODO: Should we detect and skip duplicated events?
event = Event.get(event_uuid)
# TODO: event.payload contains the whole event, not just payload - refactor
groups_ids = []
with db.session.begin_nested():
for payload_idx, payload in enumerate(event.payload['Payload']):
# TODO: marshmallow validation of all payloads
# should be done on first event ingestion (check)
relationship, errors = \
RelationshipSchema(check_existing=True).load(payload)
# Errors should never happen as the payload is validated
# with RelationshipSchema on the event ingestion
if errors:
raise MarshmallowValidationError(errors)
if relationship.id:
relationship.deleted = delete
db.session.add(relationship)
# We need ORM relationship with IDs, since Event has
# 'weak' (non-FK) relations to the objects, hence we need
# to know the ID upfront
relationship = relationship.fetch_or_create_id()
create_relation_object_events(event, relationship, payload_idx)
id_groups, version_groups = update_groups(relationship)
update_metadata(relationship, payload)
groups_ids.append(
[str(g.id) if g else g for g in id_groups + version_groups])
db.session.commit()
for ids in groups_ids:
update_indices(*ids)
def handle_event(cls,
event: dict,
no_index: bool = False,
user_id: int = None,
eager: bool = False) -> Event:
"""Handle an event payload."""
# Raises JSONSchema ValidationError
jsonschema.validate(event, EVENT_SCHEMA)
# Validate the entries in the payload
for payload in event:
errors = RelationshipSchema(check_existing=True).validate(payload)
if errors:
raise MarshmallowValidationError(errors)
event_obj = Event(payload=event,
status=EventStatus.New,
user_id=user_id)
db.session.add(event_obj)
db.session.commit()
event_uuid = str(event_obj.id)
idx_enabled = current_app.config['ASCLEPIAS_SEARCH_INDEXING_ENABLED'] \
and (not no_index)
task = process_event.s(event_uuid=event_uuid,
indexing_enabled=idx_enabled)
if eager:
task.apply(throw=True)
else:
task.apply_async()
return event_obj
def validate_payload(cls, event):
"""Validate the event payload."""
# TODO: Use invenio-jsonschemas/jsonresolver instead of this
# Validate against Event JSONSchema
# NOTE: raises `jsonschemas.ValidationError`
cls._jsonschema_validator.validate(event)
# Validate using marshmallow loader
for payload in event:
errors = RelationshipSchema(check_existing=True).validate(payload)
if errors:
raise MarshmallowValidationError(
str(errors) + "payload" + str(payload))
def process_event(event_uuid: str, indexing_enabled: bool = True):
"""Process the event."""
# TODO: Should we detect and skip duplicated events?
_set_event_status(event_uuid, EventStatus.Processing)
try:
event = Event.get(event_uuid)
groups_ids = []
with db.session.begin_nested():
for payload_idx, payload in enumerate(event.payload):
# TODO: marshmallow validation of all payloads
# should be done on first event ingestion (check)
relationship, errors = \
RelationshipSchema(check_existing=True).load(payload)
# Errors should never happen as the payload is validated
# with RelationshipSchema on the event ingestion
if errors:
raise MarshmallowValidationError(errors)
# Skip already known relationships
# NOTE: This skips any extra metadata!
if relationship.id:
continue
db.session.add(relationship)
# We need ORM relationship with IDs, since Event has
# 'weak' (non-FK) relations to the objects, hence we need
# to know the ID upfront
relationship = relationship.fetch_or_create_id()
create_relation_object_events(event, relationship, payload_idx)
id_groups, ver_groups = update_groups(relationship)
update_metadata_from_event(relationship, payload)
groups_ids.append(
[str(g.id) if g else g for g in id_groups + ver_groups])
db.session.commit()
if indexing_enabled:
compacted = compact_indexing_groups(groups_ids)
update_indices(*compacted)
_set_event_status(event_uuid, EventStatus.Done)
event_processed.send(current_app._get_current_object(), event=event)
except Exception as exc:
_set_event_status(event_uuid, EventStatus.Error)
process_event.retry(exc=exc)
def validate_id(self, id):
if self.is_create() or int(id) == int(self.instance.id):
return
raise MarshmallowValidationError('ids do not match')