class Workflow(ExecutionElement, Execution_Base):
__tablename__ = 'workflow'
playbook_id = Column(UUIDType(binary=False), ForeignKey('playbook.id'))
name = Column(String(80), nullable=False)
actions = relationship('Action', cascade='all, delete-orphan')
branches = relationship('Branch', cascade='all, delete-orphan')
start = Column(UUIDType(binary=False))
is_valid = Column(Boolean, default=False)
children = ('actions', 'branches')
__table_args__ = (UniqueConstraint('playbook_id', 'name', name='_playbook_workflow'),)
def __init__(self, name, start, id=None, actions=None, branches=None):
"""Initializes a Workflow object. A Workflow falls under a Playbook, and has many associated Actions
within it that get executed.
Args:
name (str): The name of the Workflow object.
start (int): ID of the starting Action.
id (str|UUID, optional): Optional UUID to pass into the Action. Must be UUID object or valid UUID string.
Defaults to None.
actions (list[Action]): Optional Action objects. Defaults to None.
branches (list[Branch], optional): A list of Branch objects for the Workflow object. Defaults to None.
"""
ExecutionElement.__init__(self, id)
self.name = name
self.actions = actions if actions else []
self.branches = branches if branches else []
self.start = start
self._is_paused = False
self._abort = False
self._accumulator = {branch.id: 0 for branch in self.branches}
self._execution_id = 'default'
self._instance_repo = None
self.validate()
@orm.reconstructor
def init_on_load(self):
"""Loads all necessary fields upon Workflow being loaded from database"""
self._is_paused = False
self._abort = False
self._accumulator = {branch.id: 0 for branch in self.branches}
self._instance_repo = AppInstanceRepo()
self._execution_id = 'default'
def validate(self):
"""Validates the object"""
action_ids = [action.id for action in self.actions]
errors = []
if not self.start and self.actions:
errors.append('Workflows with actions require a start parameter')
elif self.actions and self.start not in action_ids:
errors.append('Workflow start ID {} not found in actions'.format(self.start))
for branch in self.branches:
if branch.source_id not in action_ids:
errors.append('Branch source ID {} not found in workflow actions'.format(branch.source_id))
if branch.destination_id not in action_ids:
errors.append('Branch destination ID {} not found in workflow actions'.format(branch.destination_id))
self.errors = errors
self.is_valid = self._is_valid
def get_action_by_id(self, action_id):
"""Gets an Action by its ID
Args:
action_id (UUID): The ID of the Action to find
Returns:
(Action): The Action from its ID
"""
return next((action for action in self.actions if action.id == action_id), None)
def remove_action(self, action_id):
"""Removes a Action object from the Workflow's list of Actions given the Action ID.
Args:
action_id (UUID): The ID of the Action object to be removed.
Returns:
(bool): True on success, False otherwise.
"""
action_to_remove = self.get_action_by_id(action_id)
self.actions.remove(action_to_remove)
self.branches[:] = [branch for branch in self.branches if
(branch.source_id != action_id and branch.destination_id != action_id)]
logger.debug('Removed action {0} from workflow {1}'.format(action_id, self.name))
return True
def pause(self):
"""Pauses the execution of the Workflow. The Workflow will pause execution before starting the next Action"""
self._is_paused = True
logger.info('Pausing workflow {0}'.format(self.name))
def abort(self):
"""Aborts the execution of the Workflow. The Workflow will abort execution before starting the next Action"""
self._abort = True
logger.info('Aborting workflow {0}'.format(self.name))
def execute(self, execution_id, start=None, start_arguments=None, resume=False):
"""Executes a Workflow by executing all Actions in the Workflow list of Action objects.
Args:
execution_id (UUID): The UUID4 hex string uniquely identifying this workflow instance
start (int, optional): The ID of the first Action. Defaults to None.
start_arguments (list[Argument]): Argument parameters into the first Action. Defaults to None.
resume (bool, optional): Optional boolean to resume a previously paused workflow. Defaults to False.
"""
if self.is_valid:
self._execution_id = execution_id
logger.info('Executing workflow {0}'.format(self.name))
WalkoffEvent.CommonWorkflowSignal.send(self, event=WalkoffEvent.WorkflowExecutionStart)
start = start if start is not None else self.start
if not isinstance(start, UUID):
start = UUID(start)
executor = self.__execute(start, start_arguments, resume)
next(executor)
else:
logger.error('Workflow is invalid, yet executor attempted to execute.')
def __execute(self, start, start_arguments=None, resume=False):
actions = self.__actions(start=start)
for action in (action_ for action_ in actions if action_ is not None):
self._executing_action = action
logger.debug('Executing action {0} of workflow {1}'.format(action, self.name))
if self._is_paused:
self._is_paused = False
WalkoffEvent.CommonWorkflowSignal.send(self, event=WalkoffEvent.WorkflowPaused)
logger.debug('Paused workflow {} (id={})'.format(self.name, str(self.id)))
yield
if self._abort:
self._abort = False
WalkoffEvent.CommonWorkflowSignal.send(self, event=WalkoffEvent.WorkflowAborted)
logger.info('Aborted workflow {} (id={})'.format(self.name, str(self.id)))
yield
device_id = self._instance_repo.setup_app_instance(action, self)
if device_id:
result = action.execute(self._accumulator, instance=self._instance_repo.get_app_instance(device_id)(),
arguments=start_arguments, resume=resume)
else:
result = action.execute(self._accumulator, arguments=start_arguments, resume=resume)
if start_arguments:
start_arguments = None
if result and result.status == "trigger":
yield
self._accumulator[action.id] = action.get_output().result
self.__shutdown()
yield
def __actions(self, start):
current_id = start
current_action = self.get_action_by_id(current_id)
while current_action:
yield current_action
current_id = self.get_branch(current_action, self._accumulator)
current_action = self.get_action_by_id(current_id) if current_id is not None else None
yield # needed so that when for-loop calls next() it doesn't advance too far
yield # needed so you can avoid catching StopIteration exception
def get_branch(self, current_action, accumulator):
"""Executes the Branch objects associated with this Workflow to determine which Action should be
executed next.
Args:
current_action(Action): The current action that has just finished executing.
accumulator (dict): The accumulated results of previous Actions.
Returns:
(UUID): The ID of the next Action to be executed if successful, else None.
"""
if self.branches:
branches = sorted(
self.__get_branches_by_action_id(current_action.id), key=lambda branch_: branch_.priority)
for branch in branches:
# TODO: This here is the only hold up from getting rid of action._output.
# Keep whole result in accumulator
destination_id = branch.execute(current_action.get_output(), accumulator)
if destination_id is not None:
logger.debug('Branch {} with destination {} chosen by workflow {} (id={})'.format(
str(branch.id), str(destination_id), self.name, str(self.id)))
return destination_id
return None
else:
return None
def __get_branches_by_action_id(self, id_):
branches = []
if self.branches:
for branch in self.branches:
if branch.source_id == id_:
branches.append(branch)
return branches
def __shutdown(self):
# Upon finishing shut down instances
self._instance_repo.shutdown_instances()
accumulator = {str(key): value for key, value in self._accumulator.items()}
WalkoffEvent.CommonWorkflowSignal.send(self, event=WalkoffEvent.WorkflowShutdown, data=accumulator)
logger.info('Workflow {0} completed. Result: {1}'.format(self.name, self._accumulator))
def set_execution_id(self, execution_id):
"""Sets the execution UUIDD for the Workflow
Args:
execution_id (UUID): The execution ID
"""
self._execution_id = execution_id
def get_execution_id(self):
"""Gets the execution ID for the Workflow
Returns:
(UUID): The execution ID of the Workflow
"""
return self._execution_id
def get_executing_action_id(self):
"""Gets the ID of the currently executing Action
Returns:
(UUID): The ID of the currently executing Action
"""
return self._executing_action.id
def get_executing_action(self):
"""Gets the currently executing Action
Returns:
(Action): The currently executing Action
"""
return self._executing_action
def get_accumulator(self):
"""Gets the accumulator
Returns:
(dict): The accumulator
"""
return self._accumulator
def get_instances(self):
"""Gets all instances
Returns:
(list[AppInstance]): All instances
"""
return self._instance_repo.get_all_app_instances()
class Message(db.Model):
"""Flask-SqlAlchemy Table which holds messages for users.
It has a many to many relationship with both the users and roles tables.
Attributes:
id (int): The primary key
subject (str): The subject of the message
body (str): The body of the message as a JSON string
users (list[User]): The users to which this message was sent and who haven't deleted the message
roles (list[Role]): The roles to which this message was sent
workflow_execution_id (UUID): The execution id of the workflow which sent the message
requires_reauth (bool): Does the message require reauthentication to address it?
requires_response (bool): Does the message require a response?
created_at (datetime): Timestamp of message creation
history (list[MessageHistory]): The timeline of actions taken on this message
Args:
subject (str): The subject of the message
body (str): The body of the message as a JSON string
users (list[User]): The users to which this message was sent and who haven't deleted the message
roles (list[Role]): The roles to which this message was sent
requires_reauth (bool): Does the message require reauthentication to address it?
requires_response (bool): Does the message require a response?
"""
__tablename__ = 'message'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
subject = db.Column(db.String())
body = db.Column(db.String(), nullable=False)
users = db.relationship('User',
secondary=user_messages_association,
backref=db.backref('messages', lazy='dynamic'))
roles = db.relationship('Role',
secondary=role_messages_association,
backref=db.backref('messages', lazy='dynamic'))
workflow_execution_id = db.Column(UUIDType(binary=False), nullable=False)
requires_reauth = db.Column(db.Boolean, default=False)
requires_response = db.Column(db.Boolean, default=False)
created_at = db.Column(db.DateTime, default=datetime.utcnow)
history = db.relationship('MessageHistory', backref='message', lazy=True)
def __init__(self,
subject,
body,
workflow_execution_id,
users=None,
roles=None,
requires_reauth=False,
requires_response=False):
self.subject = subject
self.body = body
self.workflow_execution_id = workflow_execution_id
if not (users or roles):
message = 'Message must have users and/or roles, but has neither.'
logger.error(message)
raise ValueError(message)
self.users = users if users else []
self.roles = roles if roles else []
self.requires_reauth = requires_reauth
self.requires_response = requires_response
def record_user_action(self, user, action):
"""Records an action taken by a user on this message
Args:
user (User): The user taking the action
action (MessageAction): The action taken
"""
if user in self.users:
if ((action == MessageAction.unread
and not self.user_has_read(user)) or
(action == MessageAction.respond and
(not self.requires_response or self.is_responded()[0]))):
return
elif action == MessageAction.delete:
self.users.remove(user)
self.history.append(MessageHistory(user, action))
def user_has_read(self, user):
"""Determines if a user has read the message
Args:
user (User): The user of the query
Returns:
(bool): Has the user read the message?
"""
user_history = [
history_entry for history_entry in self.history
if history_entry.user_id == user.id
]
for history_entry in user_history[::-1]:
if history_entry.action in (MessageAction.read,
MessageAction.unread):
if history_entry.action == MessageAction.unread:
return False
if history_entry.action == MessageAction.read:
return True
else:
return False
def user_last_read_at(self, user):
"""Gets the last time the user has read the message
Args:
user (User): The user of the query
Returns:
(datetime|None): The timestamp of the last time the user has read the message or None if the message has not
been read by this user
"""
user_history = [
history_entry for history_entry in self.history
if history_entry.user_id == user.id
]
for history_entry in user_history[::-1]:
if history_entry.action == MessageAction.read:
return history_entry.timestamp
else:
return None
def get_read_by(self):
"""Gets all the usernames of the users who have read this message
Returns:
set(str): The usernames of the users who have read this message
"""
return {
entry.username
for entry in self.history if entry.action == MessageAction.read
}
def is_responded(self):
"""Has this message been responded to?
Returns:
tuple(bool, datetime|None, str|None): A tuple of if the message has been responded to, if it has then the
datetime of when it was responded to and username of the user who responded to it.
"""
if not self.requires_response:
return False, None, None
for history_entry in self.history[::-1]:
if history_entry.action == MessageAction.respond:
return True, history_entry.timestamp, history_entry.username
else:
return False, None, None
def is_authorized(self, user_id=None, role_ids=None):
"""Is a user authorized to respond to this message?
Args:
user_id (int): The ID of the user
role_ids (list[int]): The ids of the roles the user has
Returns:
(bool)
"""
if user_id:
for user in self.users:
if user_id == user.id:
return True
if role_ids:
if isinstance(role_ids, int):
role_ids = [role_ids]
for role_id in role_ids:
for role in self.roles:
if role_id == role.id:
return True
return False
def as_json(self, with_read_by=True, user=None, summary=False):
"""Gets a JSON representation of the message
Args:
with_read_by (bool, optional): Should the JSON include who has read the message? Defaults to True.
user (User, optional): If provided, information specific to the user is included.
summary (bool, optional): If True, only give a brief summary of the messsage. Defaults to False.
Returns:
"""
responded, responded_at, responded_by = self.is_responded()
ret = {
'id': self.id,
'subject': self.subject,
'created_at': utc_as_rfc_datetime(self.created_at),
'awaiting_response': self.requires_response and not responded
}
if user:
ret['is_read'] = self.user_has_read(user)
last_read_at = self.user_last_read_at(user)
if last_read_at:
ret['last_read_at'] = utc_as_rfc_datetime(last_read_at)
if not summary:
ret.update({
'body': json.loads(self.body),
'workflow_execution_id': str(self.workflow_execution_id),
'requires_reauthorization': self.requires_reauth,
'requires_response': self.requires_response
})
if responded:
ret['responded_at'] = utc_as_rfc_datetime(responded_at)
ret['responded_by'] = responded_by
if with_read_by:
ret['read_by'] = list(self.get_read_by())
return ret
class Transform(ExecutionElement, Execution_Base):
__tablename__ = 'transform'
condition_id = Column(UUIDType(binary=False), ForeignKey('condition.id', ondelete='CASCADE'))
app_name = Column(String(80), nullable=False)
action_name = Column(String(80), nullable=False)
arguments = relationship('Argument', cascade='all, delete, delete-orphan', passive_deletes=True)
children = ('arguments',)
def __init__(self, app_name, action_name, id=None, arguments=None, errors=None):
"""Initializes a new Transform object. A Transform is used to transform input into a workflow.
Args:
app_name (str): The app name associated with this transform
action_name (str): The action name for the transform.
id (str|UUID, optional): Optional UUID to pass into the Transform. Must be UUID object or valid UUID string.
Defaults to None.
arguments (list[Argument], optional): Dictionary of Argument keys to Argument values.
This dictionary will be converted to a dictionary of str:Argument. Defaults to None.
"""
ExecutionElement.__init__(self, id, errors)
self.app_name = app_name
self.action_name = action_name
self._data_param_name = None
self._api = None
self.arguments = []
if arguments:
self.arguments = arguments
self.validate()
def validate(self):
"""Validates the object"""
errors = []
try:
self._data_param_name, run, self._api = get_transform_api(self.app_name, self.action_name)
get_transform(self.app_name, run)
tmp_api = split_api_params(self._api, self._data_param_name)
validate_transform_parameters(tmp_api, self.arguments, self.action_name)
except UnknownApp:
errors.append('Unknown app {}'.format(self.app_name))
except UnknownTransform:
errors.append('Unknown transform {}'.format(self.action_name))
except InvalidArgument as e:
errors.extend(e.errors)
self.errors = errors
@orm.reconstructor
def init_on_load(self):
"""Loads all necessary fields upon Condition being loaded from database"""
if not self.errors:
errors = []
try:
self._data_param_name, run, self._api = get_transform_api(self.app_name, self.action_name)
get_transform(self.app_name, run)
except UnknownApp:
errors.append('Unknown app {}'.format(self.app_name))
except UnknownTransform:
errors.append('Unknown transform {}'.format(self.action_name))
self.errors = errors
def execute(self, action_execution_strategy, data_in, accumulator):
"""Executes the transform.
Args:
action_execution_strategy: The strategy used to execute the action (e.g. LocalActionExecutionStrategy)
data_in: The input to the condition, the last executed action of the workflow or the input to a trigger.
accumulator (dict): A record of executed actions and their results. Of form {action_name: result}.
Returns:
(obj): The transformed data
"""
original_data_in = deepcopy(data_in)
try:
arguments = self.__update_arguments_with_data(data_in)
args = validate_transform_parameters(self._api, arguments, self.action_name, accumulator=accumulator)
except InvalidArgument as e:
WalkoffEvent.CommonWorkflowSignal.send(self, event=WalkoffEvent.TransformError)
logger.error('Transform {0} has invalid input {1}. Error: {2}. '
'Returning unmodified data'.format(self.action_name, original_data_in, str(e)))
return original_data_in
try:
result = action_execution_strategy.execute(self, accumulator, args)
WalkoffEvent.CommonWorkflowSignal.send(self, event=WalkoffEvent.TransformSuccess)
return result
except ExecutionError:
logger.exception(
'Transform {0} (id={1}) encountered an error. Returning unmodified data'.format(
self.action_name, str(self.id)))
WalkoffEvent.CommonWorkflowSignal.send(self, event=WalkoffEvent.TransformError)
return original_data_in
def __update_arguments_with_data(self, data):
arguments = []
for argument in self.arguments:
if argument.name != self._data_param_name:
arguments.append(argument)
arguments.append(Argument(self._data_param_name, value=data))
return arguments
class ActionStatus(Device_Base):
"""ORM for an Action event in the database
"""
__tablename__ = 'action_status'
execution_id = Column(UUIDType(binary=False), primary_key=True)
action_id = Column(UUIDType(binary=False), nullable=False)
name = Column(String, nullable=False)
app_name = Column(String, nullable=False)
action_name = Column(String, nullable=False)
result = Column(String)
arguments = Column(String)
status = Column(Enum(ActionStatusEnum), nullable=False)
started_at = Column(DateTime, default=datetime.utcnow)
completed_at = Column(DateTime)
_workflow_status_id = Column(UUIDType(binary=False),
ForeignKey('workflow_status.execution_id'))
def __init__(self,
execution_id,
action_id,
name,
app_name,
action_name,
arguments=None):
self.execution_id = execution_id
self.action_id = action_id
self.name = name
self.app_name = app_name
self.action_name = action_name
self.arguments = arguments
self.status = ActionStatusEnum.executing
def aborted(self):
if self.status == ActionStatusEnum.awaiting_data:
self.status = ActionStatusEnum.aborted
def running(self):
self.status = ActionStatusEnum.executing
def awaiting_data(self):
self.status = ActionStatusEnum.awaiting_data
def completed_success(self, data):
self.status = ActionStatusEnum.success
self.result = json.dumps(data['result'])
self.completed_at = datetime.utcnow()
def completed_failure(self, data):
self.status = ActionStatusEnum.failure
self.result = json.dumps(data['result'])
self.completed_at = datetime.utcnow()
def as_json(self, summary=False):
ret = {
"execution_id": str(self.execution_id),
"action_id": str(self.action_id),
"name": self.name,
"app_name": self.app_name,
"action_name": self.action_name
}
if summary:
return ret
ret.update({
"arguments":
json.loads(self.arguments) if self.arguments else [],
"status":
self.status.name,
"started_at":
utc_as_rfc_datetime(self.started_at)
})
if self.status in [ActionStatusEnum.success, ActionStatusEnum.failure]:
ret["result"] = json.loads(self.result)
ret["completed_at"] = utc_as_rfc_datetime(self.completed_at)
return ret
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy_utils import UUIDType
import uuid
class Config(object):
SQLALCHEMY_DATABASE_URI = 'postgresql://*****:*****@postgres:5432/gamehive'
app = Flask(__name__)
app.config.from_object(Config)
db = SQLAlchemy(app)
items_table = db.Table(
'items',
db.Column('player_id', UUIDType(binary=False),
db.ForeignKey('player.uid')),
db.Column('item_id', UUIDType(binary=False), db.ForeignKey('item.uid')))
class Player(db.Model):
uid = db.Column(UUIDType(binary=False),
primary_key=True,
default=uuid.uuid4)
nickname = db.Column(db.String(80), nullable=False)
email = db.Column(db.String(120), unique=True, nullable=False)
skill = db.Column(db.Integer, nullable=False)
class TesTask(CRUDMixin, db.Model):
id = db.Column(UUIDType(binary=False), primary_key=True, default=lambda: str(uuid.uuid4()))
user_id = db.Column(db.String(64), nullable=True)
tenant_id = db.Column(db.String(64), nullable=True)
backend_id = db.Column(db.String(64), nullable=False)
task_json = db.Column(NestedMutableJson)
state = db.Column(db.Enum(TaskStatus), nullable=False)
created_ts = db.Column(
db.DateTime(timezone=True),
default=datetime.datetime.utcnow
)
updated_ts = db.Column(
db.DateTime(timezone=True),
default=datetime.datetime.utcnow,
onupdate=datetime.datetime.utcnow
)
@hybrid_property
def creation_time(self):
return self.created_ts
@creation_time.setter
def creation_time(self, value):
self.created_ts = value
@hybrid_property
def resources(self):
return TesResourcesSchema().load(self.task_json["resources"]).data
@resources.setter
def resources(self, value):
self.task_json["resources"] = TesResourcesSchema().dump(value).data
@hybrid_property
def executors(self):
return TesExecutorSchema().load(self.task_json["executors"], many=True).data
# FIXME: .append() fails to trigger setter on list hybrid properties
# https://groups.google.com/forum/#!topic/sqlalchemy/HZKTuD36Drs
@executors.setter
def executors(self, value):
self.task_json["executors"] = TesExecutorSchema().dump(value, many=True).data
@hybrid_property
def name(self):
return (self.task_json["name"])
@name.setter
def name(self, value):
self.task_json["name"] = value
@hybrid_property
def description(self):
return (self.task_json["description"])
@description.setter
def description(self, value):
self.task_json["description"] = value
@hybrid_property
def inputs(self):
return TesInputSchema().load(self.task_json["inputs"], many=True).data
@inputs.setter
def inputs(self, value):
self.task_json["inputs"] = TesInputSchema().dump(value, many=True).data
@hybrid_property
def outputs(self):
return TesOutputSchema().load(self.task_json["outputs"], many=True).data
@outputs.setter
def outputs(self, value):
self.task_json["outputs"] = TesOutputSchema().dump(value, many=True).data
@hybrid_property
def logs(self):
return TaskLogSchema().load(self.task_json["logs"], many=True).data
@logs.setter
def logs(self, value):
self.task_json["logs"] = TaskLogSchema().dump(value, many=True).data
@hybrid_property
def tags(self):
return self.task_json["tags"]
@tags.setter
def tags(self, value):
self.task_json["tags"] = value
def __init__(self,
resources: TesResources = TesResources(), executors: List[TesExecutor] = [],
id: str = None, state: TaskStatus = TaskStatus.UNKNOWN,
name: str = "", description: str = "", inputs: List[TesInput] = [],
outputs: List[TesOutput] = [], tags: dict = {},
volumes: List[str] = [], logs: List[TaskLogSchema] = [],
creation_time: datetime = datetime.datetime.utcnow):
self.task_json = TesTaskSchema().dump({
"resources": resources,
"executors": executors,
"name": name,
"description": description,
"inputs": inputs,
"outputs": outputs,
"logs": logs,
"tags": tags
}).data
self.state = state
class BaseMixin(object):
id = db.Column(
UUIDType(binary=False), primary_key=True, default=fields.gen_uuid
)
# Audit fields
created_at = db.Column(db.DateTime, default=datetime.datetime.utcnow)
updated_at = db.Column(
db.DateTime,
default=datetime.datetime.utcnow,
onupdate=datetime.datetime.utcnow,
)
def __repr__(self):
"""
String representation based on type and name by default.
"""
return "<%s %s>" % (type(self).__name__, self.name)
@classmethod
def query(cls):
"""
Shorthand to access session query object.
"""
return db.session.query(cls)
@classmethod
def get(cls, id):
"""
Shorthand to retrieve data by id.
"""
return cls.query.get(id)
@classmethod
def get_by(cls, **kw):
"""
Shorthand to retrieve data by using filters. It returns the first
element of the returned data.
"""
return cls.query.filter_by(**kw).first()
@classmethod
def get_all(cls):
"""
Shorthand to retrieve all data for a model.
"""
return cls.query.all()
@classmethod
def get_all_by(cls, **kw):
"""
Shorthand to retrieve data by using filters.
"""
return cls.query.filter_by(**kw).all()
@classmethod
def create(cls, **kw):
"""
Shorthand to create an entry via the database session.
"""
instance = cls(**kw)
try:
db.session.add(instance)
db.session.commit()
except:
db.session.rollback()
db.session.remove()
raise
return instance
@classmethod
def create_no_commit(cls, **kw):
"""
Shorthand to create an entry via the database session without commiting
the request.
"""
instance = cls(**kw)
db.session.add(instance)
return instance
@classmethod
def delete_all_by(cls, **kw):
"""
Shorthand to delete data by using filters.
"""
result = cls.query.filter_by(**kw).delete()
db.session.commit()
return result
@classmethod
def get_id_map(cls, field="shotgun_id"):
"""
Build a map to easily match a field value with an id. It's useful during
mass import to build foreign keys.
"""
entry_map = {}
entries = cls.query.all()
for entry in entries:
entry_map[getattr(entry, field)] = entry.id
return entry_map
@classmethod
def create_from_import(cls, data):
"""
Create a new instance of the model based on data that comes from the Zou
API.
"""
if "type" in data:
del data["type"]
previous_data = cls.get(data["id"])
if previous_data is None:
return cls.create(**data)
else:
previous_data.update(data)
return previous_data
@classmethod
def create_from_import_list(cls, data_list):
"""
Create a list of instances of the model based on data that comes from
the Zou API.
"""
for data in data_list:
cls.create_from_import(data)
@classmethod
def delete_from_import(cls, instance_id):
"""
Delete an entry and its related base on the entry id.
"""
instance = cls.get(instance_id)
if instance is not None:
instance.delete()
return instance_id
@classmethod
def commit(cls):
db.session.commit()
def save(self):
"""
Shorthand to create an entry via the database session based on current
instance fields.
"""
try:
self.updated_at = datetime.datetime.now()
db.session.add(self)
db.session.commit()
except:
db.session.rollback()
db.session.remove()
raise
def delete(self):
"""
Shorthand to delete an entry via the database session based on current
instance id.
"""
try:
db.session.delete(self)
db.session.commit()
except:
db.session.rollback()
db.session.remove()
raise
def delete_no_commit(self):
"""
Shorthand to delete an entry via the database session based on current
instance id. The change is not commited.
"""
db.session.delete(self)
return True
def update(self, data):
"""
Shorthand to update an entry via the database session based on current
instance fields.
"""
try:
self.updated_at = datetime.datetime.now()
for key, value in data.items():
setattr(self, key, value)
db.session.add(self)
db.session.commit()
except:
db.session.rollback()
db.session.remove()
raise
def set_links(self, ids, LinkTable, field_left, field_right):
for id in ids:
link = LinkTable.query.filter_by(
kwargs={field_left: self.id, field_right: id}
).first()
if link is None:
link = LinkTable(kwargs={field_left: self.id, field_right: id})
db.session.add(link)
db.session.commit()
class ActionStatus(Execution_Base):
"""ORM for an Action event in the database
Attributes:
execution_id (UUID): The execution ID of the Action
action_id (UUID): The ID of the Action
name (str): The name of the Action
app_name (str): The App name for the Action
action_name (str): The Action name for the Action
result (str): The result of the Action
arguments (str): The Arguments for the Action, in string representation
status (ActionStatusEnum): The status of the Action
started_at (datetime): The time the Action started
completed_at (datetime): The time the Action completed
_workflow_status_id (UUID): The FK ID of the WorkflowStatus
"""
__tablename__ = 'action_status'
execution_id = Column(UUIDType(binary=False), primary_key=True)
action_id = Column(UUIDType(binary=False), nullable=False)
name = Column(String, nullable=False)
app_name = Column(String, nullable=False)
action_name = Column(String, nullable=False)
result = Column(String)
arguments = Column(String)
status = Column(Enum(ActionStatusEnum, name='ActionStatusEnum'),
nullable=False)
started_at = Column(DateTime, default=datetime.utcnow)
completed_at = Column(DateTime)
_workflow_status_id = Column(UUIDType(binary=False),
ForeignKey('workflow_status.execution_id'))
def __init__(self,
execution_id,
action_id,
name,
app_name,
action_name,
arguments=None):
self.execution_id = execution_id
self.action_id = action_id
self.name = name
self.app_name = app_name
self.action_name = action_name
self.arguments = arguments
self.status = ActionStatusEnum.executing
def aborted(self):
"""Sets status to aborted"""
if self.status == ActionStatusEnum.awaiting_data:
self.status = ActionStatusEnum.aborted
def running(self):
"""Sets status to running"""
self.status = ActionStatusEnum.executing
def awaiting_data(self):
"""Sets status to awaiting data"""
self.status = ActionStatusEnum.awaiting_data
def completed_success(self, data):
"""Sets status to completed successfully"""
self.status = ActionStatusEnum.success
self.result = json.dumps(data['result'])
self.completed_at = datetime.utcnow()
def completed_failure(self, data):
"""Sets status to completed unsuccessfully"""
self.status = ActionStatusEnum.failure
self.result = json.dumps(data['result'])
self.completed_at = datetime.utcnow()
def as_json(self, summary=False):
"""Gets the JSON representation of the object
Args:
summary (bool, optional): Only get a limited JSON? Defaults to False
Returns:
(dict): The JSON representation of the object
"""
ret = {
"execution_id": str(self.execution_id),
"action_id": str(self.action_id),
"name": self.name,
"app_name": self.app_name,
"action_name": self.action_name
}
if summary:
return ret
ret.update({
"arguments":
json.loads(self.arguments) if self.arguments else [],
"status":
self.status.name,
"started_at":
utc_as_rfc_datetime(self.started_at)
})
if self.status in [ActionStatusEnum.success, ActionStatusEnum.failure]:
ret["result"] = json.loads(self.result)
ret["completed_at"] = utc_as_rfc_datetime(self.completed_at)
return ret
class WorkflowStatus(Execution_Base):
"""Case ORM for a Workflow event in the database
Attributes:
execution_id (UUID): Execution ID of the Workflow
workflow_id (UUID): ID of the Workflow
name (str): Name of the Workflow
status (str): Status of the Workflow
started_at (datetime): Time the Workflow started
completed_at (datetime): Time the Workflow ended
_action_statuses (list[ActionStatus]): A list of ActionStatus objects for this WorkflowStatus
"""
__tablename__ = 'workflow_status'
execution_id = Column(UUIDType(binary=False), primary_key=True)
workflow_id = Column(UUIDType(binary=False), nullable=False)
name = Column(String, nullable=False)
status = Column(Enum(WorkflowStatusEnum, name='WorkflowStatusEnum'),
nullable=False)
started_at = Column(DateTime)
completed_at = Column(DateTime)
_action_statuses = relationship('ActionStatus',
backref=backref('_workflow_status'),
cascade='all, delete-orphan')
def __init__(self, execution_id, workflow_id, name):
self.execution_id = execution_id
self.workflow_id = workflow_id
self.name = name
self.status = WorkflowStatusEnum.pending
def running(self):
"""Sets the status to running"""
self.started_at = datetime.utcnow()
self.status = WorkflowStatusEnum.running
def paused(self):
"""Sets the status to paused"""
self.status = WorkflowStatusEnum.paused
def awaiting_data(self):
"""Sets the status to awaiting data"""
self.status = WorkflowStatusEnum.awaiting_data
if self._action_statuses:
self._action_statuses[-1].awaiting_data()
def completed(self):
"""Sets the status to completed"""
self.completed_at = datetime.utcnow()
self.status = WorkflowStatusEnum.completed
def aborted(self):
"""Sets the status to aborted"""
self.completed_at = datetime.utcnow()
self.status = WorkflowStatusEnum.aborted
if self._action_statuses:
self._action_statuses[-1].aborted()
def add_action_status(self, action_status):
"""Adds an ActionStatus
Args:
action_status (ActionStatus): The ActionStatus to add
"""
self._action_statuses.append(action_status)
def as_json(self, full_actions=False):
"""Gets the JSON representation of the WorkflowStatus
Args:
full_actions (bool, optional): Get the full Action objects as well? Defaults to False
Returns:
The JSON representation of the object
"""
ret = {
"execution_id": str(self.execution_id),
"workflow_id": str(self.workflow_id),
"name": self.name,
"status": self.status.name
}
if self.started_at:
ret["started_at"] = utc_as_rfc_datetime(self.started_at)
if self.status in [
WorkflowStatusEnum.completed, WorkflowStatusEnum.aborted
]:
ret["completed_at"] = utc_as_rfc_datetime(self.completed_at)
if full_actions:
ret["action_statuses"] = [
action_status.as_json()
for action_status in self._action_statuses
]
elif self._action_statuses and self.status != WorkflowStatusEnum.completed:
current_action = self._action_statuses[-1]
ret['current_action'] = current_action.as_json(summary=True)
return ret
class ImportMixin:
id = sa.Column(sa.Integer, primary_key=True)
uuid = sa.Column(UUIDType(binary=True), primary_key=False, default=uuid4)
import sys
from sqlalchemy_utils import UUIDType, EmailType, LocaleType, TimezoneType
from sqlalchemy.dialects.postgresql import JSONB
from pytz import timezone as pytz_timezone
from babel import Locale
from zou.app import db
from zou.app.models.serializer import SerializerMixin
from zou.app.models.base import BaseMixin
department_link = db.Table(
"department_link",
db.Column("person_id", UUIDType(binary=False), db.ForeignKey("person.id")),
db.Column("department_id", UUIDType(binary=False),
db.ForeignKey("department.id")),
)
class Person(db.Model, BaseMixin, SerializerMixin):
"""
Describe a member of the studio (and an API user).
"""
first_name = db.Column(db.String(80), nullable=False)
last_name = db.Column(db.String(80), nullable=False)
email = db.Column(EmailType, unique=True)
phone = db.Column(db.String(30))
active = db.Column(db.Boolean(), default=True)
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy_utils import UUIDType
from depc.extensions import db
from depc.models import BaseModel
users_news_association_table = db.Table(
"users_news",
db.Column(
"user_id", UUIDType(binary=False), db.ForeignKey("users.id"), primary_key=True
),
db.Column(
"news_id", UUIDType(binary=False), db.ForeignKey("news.id"), primary_key=True
),
UniqueConstraint("user_id", "news_id", name="users_news_uix"),
)
class News(BaseModel):
__tablename__ = "news"
title = db.Column(db.String(100), nullable=False)
message = db.Column(db.String(), nullable=False)
users = db.relationship(
"User", backref=db.backref("news", uselist=True), secondary="users_news"
)
class Result(db.Model):
__tablename_ = 'Result'
componentId = db.Column(UUIDType(binary=True),
db.ForeignKey(Component.Id),
nullable=False)
timeStamp = db.Column(db.DATETIME,
db.ForeignKey(Data.timeStamp),
nullable=False)
resultTimeStamp = db.Column(db.DATETIME, primary_key=True, nullable=False)
data = db.Column(db.NVARCHAR)
def json(self):
return {
'componentId': str(self.componentId),
'timeStamp': str(self.timeStamp),
'resultTimeStamp': str(self.resultTimeStamp),
'data': self.data
}
def add_data(_timestamp, _resultTimeStamp, _componentId, _data):
new_data = Result(componentId=_componentId,
timeStamp=_timestamp,
resultTimeStamp=_resultTimeStamp,
data=_data)
db.session.add(new_data)
db.session.commit()
def get_all_data():
return [Result.json(data) for data in Result.query.all()]
def get_data_by_id(_componentId):
return [
Result.json(data)
for data in Result.query.filter_by(componentId=_componentId).
order_by(desc(Result.timeStamp)).all()
]
def get_data_by_timestamp(_componentId, _timeStamp):
return Result.json(
Result.query.filter_by(componentId=_componentId,
timeStamp=_timeStamp).all())
def get_data_the_lastrows(counts, _componentId):
return [
Result.json(data)
for data in Result.query.filter_by(componentId=_componentId).
order_by(desc(Result.timeStamp)).limit(counts).all()
]
def get_data_allrows(_componentId):
return [
Result.json(data)
for data in Result.query.filter_by(componentId=_componentId).
order_by(desc(Result.timeStamp)).all()
]
def delete_data(_resultTimeStamp):
Result.query.filter_by(timeStamp=_resultTimeStamp).delete()
db.session.commit()
def __repr__(self):
data_object = {
'componentId': self.componentId,
'timeStamp': str(self.timeStamp),
'resultTimeStamp': str(self.resultTimeStamp),
'data': self.data
}
return json.dumps(data_object)
class Entity(db.Model, BaseMixin, SerializerMixin):
"""
Base model to represent assets, shots, sequences, episodes and scenes.
They have different meaning but they share the same behaviour toward
tasks and files.
"""
id = db.Column(
UUIDType(binary=False),
primary_key=True,
default=fields.gen_uuid
)
name = db.Column(db.String(160), nullable=False)
description = db.Column(db.String(600))
shotgun_id = db.Column(db.Integer)
canceled = db.Column(db.Boolean, default=False)
project_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("project.id"),
nullable=False,
index=True
)
entity_type_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("entity_type.id"),
nullable=False,
index=True
)
parent_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("entity.id"),
index=True
) # sequence or episode
source_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("entity.id"),
index=True,
nullable=True
) # if the entity is generated from another one (like shots from scene).
preview_file_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("preview_file.id", name="fk_main_preview")
)
data = db.Column(JSONB)
entities_out = db.relationship(
"Entity",
secondary="entity_link",
primaryjoin=(id == EntityLink.entity_in_id),
secondaryjoin=(id == EntityLink.entity_out_id),
backref="entities_in"
)
instance_casting = db.relationship(
"AssetInstance",
secondary="asset_instance_link",
backref="shots"
)
__table_args__ = (
db.UniqueConstraint(
"name",
"project_id",
"entity_type_id",
"parent_id",
name="entity_uc"
),
)
class DomainModel(db.Model):
id = db.Column(UUIDType(), primary_key=True)
user = db.Column(UUIDType())
name = db.Column(db.Unicode(40))
pathname = db.Column(db.UnicodeText)
posted_at = db.Column(db.DateTime)
class Element(Base, BaseForItems):
__tablename__ = 'Elements'
value = Column(String, nullable=False)
article_id = Column(UUIDType(binary=False), ForeignKey('Article.id'))
article = relationship("Article", backref="elements")
class Branch(ExecutionElement, Execution_Base):
__tablename__ = 'branch'
workflow_id = Column(UUIDType(binary=False),
ForeignKey('workflow.id', ondelete='CASCADE'))
source_id = Column(UUIDType(binary=False), nullable=False)
destination_id = Column(UUIDType(binary=False), nullable=False)
status = Column(String(80))
condition = relationship('ConditionalExpression',
cascade='all, delete-orphan',
uselist=False,
passive_deletes=True)
priority = Column(Integer)
children = ('condition', )
def __init__(self,
source_id,
destination_id,
id=None,
status='Success',
condition=None,
priority=999):
"""Initializes a new Branch object.
Args:
source_id (int): The ID of the source action that will be sending inputs to this Branch.
destination_id (int): The ID of the destination action that will be returned if the conditions for this
Branch are met.
id (str|UUID, optional): Optional UUID to pass into the Action. Must be UUID object or valid UUID string.
Defaults to None.
status (str, optional): Optional field to keep track of the status of the Branch. Defaults to
"Success".
condition (ConditionalExpression, optional): The condition which must be fulfilled for this branch.
Defaults to None.
priority (int, optional): Optional priority parameter to specify which Branch in the Workflow's
list of Branches should be executed if multiple have conditions resulting to True.
Defaults to 999 (lowest priority).
"""
ExecutionElement.__init__(self, id)
self.source_id = source_id
self.destination_id = destination_id
self.status = status
self.priority = priority
self.condition = condition
self._counter = 0
self.validate()
@orm.reconstructor
def init_on_load(self):
"""Loads all necessary fields upon Branch being loaded from database"""
self._counter = 0
def validate(self):
pass
def execute(self, action_execution_strategy, status, current_action,
accumulator):
"""Executes the Branch object, determining if this Branch should be taken.
Args:
action_execution_strategy: The strategy used to execute the action (e.g. LocalActionExecutionStrategy)
status (str): The status of the current action
current_action (Action): The previously-executed Action.
accumulator (dict): The accumulated data from previous Actions.
Returns:
(UUID): Destination UID for the next Action that should be taken, None if the data_in was not valid
for this Branch.
"""
logger.debug('Executing branch {}'.format(str(self.id)))
self._counter += 1
accumulator[self.id] = self._counter
if current_action is not None and status == self.status:
data_in = accumulator[current_action.id]
if self.condition is None or self.condition.execute(
action_execution_strategy,
data_in=data_in,
accumulator=accumulator):
WalkoffEvent.CommonWorkflowSignal.send(
self, event=WalkoffEvent.BranchTaken)
logger.debug('Branch is valid for input {0}'.format(data_in))
return self.destination_id
else:
logger.debug(
'Branch is not valid for input {0}'.format(data_in))
WalkoffEvent.CommonWorkflowSignal.send(
self, event=WalkoffEvent.BranchNotTaken)
return None
else:
return None
class Article(Base, BaseForItems):
__tablename__ = 'Article'
notes = Column(Text, default="")
project_id = Column(UUIDType(binary=False), ForeignKey('Project.id'))
project = relationship("Project", backref="articles")
class Project(CRUDMixin, db.Model):
__tablename__ = 'projects'
id = db.Column(db.Integer(), primary_key=True)
github_id = db.Column(db.Integer(), unique=True, nullable=True)
uuid = db.Column(UUIDType(),
nullable=False,
unique=True,
index=True,
default=uuid.uuid4)
title = db.Column(db.String(MAX_LENGTH_TITLE), nullable=False)
description = db.Column(db.String(MAX_LENGTH_DESCRIPTION), nullable=False)
created_at = db.Column(db.DateTime,
nullable=False,
default=dt.datetime.utcnow)
status = db.Column(db.Enum(Status_), nullable=False)
contributors = db.relationship('User',
secondary=users_projects,
backref=db.backref('contribution_projects',
lazy='dynamic'))
pref_skills = db.relationship('Skill',
secondary=skills_projects,
backref=db.backref('projects',
lazy='dynamic'))
user_id = db.Column(db.Integer, db.ForeignKey('users.id'), nullable=False)
@classmethod
def from_project_form(cls, form, validate=False):
""" Creates and returns a project object from a NewProjectForm
:param form: PEC.project.forms.NewProjectForm
:param validate: Whether the form should be validated before creating object, default False
:return: Project object or None if validation fails
"""
if validate:
if not form.validate():
return None
project = cls()
project.title = form.title.data
project.description = form.description.data
return project
def mark_proposal(self, commit=True):
""" Changes project status to Proposal #TODO Project status
:param commit: Change to be committed to database, defaults to True
:return:
"""
self.update(status=Status_.proposal, commit=commit)
def mark_development(self, commit=True):
""" Changes project status to Development #TODO Project status
:param commit: Changes to be committed to database, defaults to True
:return:
"""
self.update(status=Status_.development, commit=commit)
def add_pref_skills(self, *args):
""" Adds given user skill attributes to the list or preferred skills in the project
Valid skills enumerated in PEC.user.attributes.Skill
Usage:
project.add_pref_skills('HTML', 'CSS')
skills_to_add = ['HTML', 'CSS']
project.add_pref_skills(*skills_to_add)
:param args: Skills (as strings)
:raises KeyError if invalid skill provided
:return: None
"""
for skill in args:
skill_ = Skill.get(name=skill)
if skill_ not in self.pref_skills:
self.pref_skills.append(skill_)
def has_pref_skills(self, *args):
""" Checks if project contains all given preferred skills
Valid skills enumerated in PEC.user.attributes.Skill
Usage:
if project.has_pref_skills('HTML', 'CSS'):
print('The project prefers HTML and CSS skills')
skills_to_check = ['HTML', 'CSS']
if project.has_pref_skill(*skills_to_check):
print('The project prefers HTML and CSS skills')
:param args: PEC.user.attributes.Skill
:raises KeyError if invalid skill provided
:return: True if ALL skills are preferred, False otherwise
"""
for skill in args:
skill_ = Skill.get(name=skill)
if skill_ not in self.pref_skills:
return False
return True
def __repr__(self):
return '<Project {}>'.format(self.uuid)
class BaseForItems():
id = Column(UUIDType(binary=False), primary_key=True, default=uuid.uuid4)
name = Column(String, nullable=False)
class WorkflowStatus(Device_Base):
"""Case ORM for a Workflow event in the database
"""
__tablename__ = 'workflow_status'
execution_id = Column(UUIDType(binary=False), primary_key=True)
workflow_id = Column(UUIDType(binary=False), nullable=False)
name = Column(String, nullable=False)
status = Column(Enum(WorkflowStatusEnum), nullable=False)
started_at = Column(DateTime)
completed_at = Column(DateTime)
_action_statuses = relationship('ActionStatus',
backref=backref('_workflow_status'),
cascade='all, delete-orphan')
def __init__(self, execution_id, workflow_id, name):
self.execution_id = execution_id
self.workflow_id = workflow_id
self.name = name
self.status = WorkflowStatusEnum.pending
def running(self):
self.started_at = datetime.utcnow()
self.status = WorkflowStatusEnum.running
def paused(self):
self.status = WorkflowStatusEnum.paused
def awaiting_data(self):
self.status = WorkflowStatusEnum.awaiting_data
if self._action_statuses:
self._action_statuses[-1].awaiting_data()
def completed(self):
self.completed_at = datetime.utcnow()
self.status = WorkflowStatusEnum.completed
def aborted(self):
self.completed_at = datetime.utcnow()
self.status = WorkflowStatusEnum.aborted
if self._action_statuses:
self._action_statuses[-1].aborted()
def add_action_status(self, action_status):
self._action_statuses.append(action_status)
def as_json(self, full_actions=False):
ret = {
"execution_id": str(self.execution_id),
"workflow_id": str(self.workflow_id),
"name": self.name,
"status": self.status.name
}
if self.started_at:
ret["started_at"] = utc_as_rfc_datetime(self.started_at)
if self.status in [
WorkflowStatusEnum.completed, WorkflowStatusEnum.aborted
]:
ret["completed_at"] = utc_as_rfc_datetime(self.completed_at)
if full_actions:
ret["action_statuses"] = [
action_status.as_json()
for action_status in self._action_statuses
]
elif self._action_statuses and self.status != WorkflowStatusEnum.completed:
current_action = self._action_statuses[-1]
ret['current_action'] = current_action.as_json(summary=True)
return ret
class CreationMixin():
id = Column(UUIDType(binary=False), primary_key=True, unique=True)
@classmethod
def add(cls, **kwargs):
####################################################
# remove any keys in the payload that don't belong
bad_keys = []
for key in kwargs:
if not key in cls.__dict__:
bad_keys.append(key)
for key in bad_keys:
del kwargs[key]
####################################################
thing = cls(**kwargs)
if thing.id is None:
thing.id = str(uuid4())
DBSession.add(thing)
DBSession.commit()
return thing
@classmethod
def get_all(cls):
things = DBSession.query(cls, ).all()
return things
@classmethod
def get_paged(cls, start=0, count=25):
things = DBSession.query(cls, ).slice(start, start + count).all()
return things
@classmethod
def get_by_id(cls, id):
thing = DBSession.query(cls, ).filter(cls.id == id, ).first()
return thing
@classmethod
def delete_by_id(cls, id):
thing = cls.get_by_id(id)
if thing is not None:
DBSession.delete(thing)
DBSession.commit()
return thing
@classmethod
def update_by_id(cls, id, **kwargs):
keys = set(cls.__dict__)
thing = DBSession.query(cls).filter(cls.id == id).first()
if thing is not None:
for k in kwargs:
if k in keys:
setattr(thing, k, kwargs[k])
thing.modified_datetime = datetime.datetime.now()
DBSession.add(thing)
DBSession.commit()
return thing
@classmethod
def reqkeys(cls):
keys = []
for key in cls.__table__.columns:
if '__required__' in type(key).__dict__:
keys.append(str(key).split('.')[1])
return keys
def to_dict(self):
return {
'id': str(self.id),
'creation_datetime': str(self.creation_datetime),
}
class Project(db.Model, BaseMixin, SerializerMixin):
"""
Describes a CG production the studio works on.
"""
name = db.Column(db.String(80), nullable=False, unique=True, index=True)
code = db.Column(db.String(80))
description = db.Column(db.Text())
shotgun_id = db.Column(db.Integer)
file_tree = db.Column(JSONB)
data = db.Column(JSONB)
has_avatar = db.Column(db.Boolean(), default=False)
fps = db.Column(db.String(10))
ratio = db.Column(db.String(10))
resolution = db.Column(db.String(12))
production_type = db.Column(db.String(20), default="short")
start_date = db.Column(db.Date())
end_date = db.Column(db.Date())
man_days = db.Column(db.Integer)
nb_episodes = db.Column(db.Integer, default=0)
episode_span = db.Column(db.Integer, default=0)
project_status_id = db.Column(
UUIDType(binary=False), db.ForeignKey("project_status.id"), index=True
)
team = db.relationship("Person", secondary="project_person_link")
asset_types = db.relationship(
"EntityType", secondary="project_asset_type_link"
)
task_statuses = db.relationship(
"TaskStatus", secondary="project_task_status_link"
)
task_types = db.relationship("TaskType", secondary="project_task_type_link")
def set_team(self, person_ids):
for person_id in person_ids:
link = ProjectPersonLink.query.filter_by(
project_id=self.id, person_id=person_id
).first()
if link is None:
link = ProjectPersonLink(
project_id=self.id, person_id=person_id
)
db.session.add(link)
db.session.commit()
def set_task_types(self, task_type_ids):
return self.set_links(
task_type_ids, ProjectTaskTypeLink, "project_id", "task_type_id"
)
def set_task_statuses(self, task_status_ids):
return self.set_links(
task_status_ids,
ProjectTaskStatusLink,
"project_id",
"task_status_id",
)
def set_asset_types(self, asset_type_ids):
return self.set_links(
asset_type_ids, ProjectAssetTypeLink, "project_id", "entity_type_id"
)
@classmethod
def create_from_import(cls, data):
is_update = False
previous_project = cls.get(data["id"])
person_ids = data.get("team", None)
task_type_ids = data.get("task_types", None)
task_status_ids = data.get("task_statuses", None)
data.pop("team", None)
data.pop("type", None)
data.pop("project_status_name", None)
if previous_project is None:
previous_project = cls.create(**data)
previous_project.save()
else:
is_update = True
previous_project.update(data)
previous_project.save()
if person_ids is not None:
previous_project.set_team(person_ids)
if task_type_ids is not None:
previous_project.set_task_types(task_type_ids)
if task_status_ids is not None:
previous_project.set_task_statuses(task_status_ids)
return (previous_project, is_update)
class Template(db.Model):
__tablename__ = 'templates'
uuid = db.Column(UUIDType(binary=False), primary_key=True, default=uuid.uuid4())
timestamp = db.Column(db.DateTime, index=True, default=datetime.utcnow())
name = db.Column(db.String(64), unique=True, nullable=False)
description = db.Column(db.Text, nullable=True)
vcpus = db.Column(db.Integer, nullable=False)
memory = db.Column(db.Integer, nullable=False)
xml_path = db.Column(db.String(255), nullable=False)
images_path = db.Column(db.PickleType, nullable=False)
def __init__(self, data):
self.uuid = uuid.uuid4()
self.name = data['name']
self.description = data['description']
self.vcpus = data['vcpus']
self.memory = round(data['memory'] * 0.000976562) # KiB to MiB
self.xml_path = data['xml_path']
self.images_path = pickle.dumps(data['images_path'])
def save(self):
try:
db.session.add(self)
db.session.commit()
except KeyError or IntegrityError as e:
db.session.rollback()
raise e
@staticmethod
def get(template_uuid=None):
return Template.query.get(template_uuid) if template_uuid else Template.query.all()
def update(self, data):
if 'name' in data and data['name'] != "":
self.name = data['name']
if 'description' in data and data['description'] != "":
self.description = data['description']
try:
db.session.commit()
except Exception as e:
db.session.rollback()
raise e
@staticmethod
def delete(template):
try:
db.session.delete(template)
db.session.commit()
except UnmappedInstanceError as e:
db.session.rollback()
raise e
def to_dict(self):
return dict(uuid=str(self.uuid),
timestamp=self.timestamp,
name=self.name,
description=self.description,
vcpus=self.vcpus,
memory=self.memory,
xml_path=self.xml_path,
images_path=pickle.loads(self.images_path))
class Item(db.Model):
uid = db.Column(UUIDType(binary=False),
primary_key=True,
default=uuid.uuid4)
skill = db.Column(db.Integer, nullable=False)
class Lab(db.Model):
__tablename__ = 'labs'
uuid = db.Column(UUIDType(binary=False), primary_key=True, default=uuid.uuid4())
code = db.Column(db.String(64), unique=True, nullable=False)
description = db.Column(db.Text, nullable=False)
start_ip_range = db.Column(IPAddressType, unique=True, nullable=False)
end_ip_range = db.Column(IPAddressType, unique=True, nullable=False)
hosts_vcpus = db.Column(db.Integer, nullable=False)
hosts_memory = db.Column(db.Integer, nullable=False)
hosts_disk = db.Column(db.Integer, nullable=False)
hosts = db.relationship('Host', backref='lab', lazy=True, cascade="all, delete-orphan")
def __init__(self, data):
self.uuid = uuid.uuid4()
self.code = data['code'].upper()
self.description = data['description']
self.start_ip_range = ipaddress.ip_address(data['start_ip_range'])
self.end_ip_range = ipaddress.ip_address(data['end_ip_range'])
self.hosts_vcpus = data['hosts']['vcpus']
self.hosts_memory = data['hosts']['memory']
self.hosts_disk = data['hosts']['disk']
def save(self):
"""Adds into database an existent laboratory
:return: None
"""
try:
db.session.add(self)
db.session.commit()
except KeyError or IntegrityError as e:
db.session.rollback()
raise e
def remove(self):
"""Deletes from database an existent laboratory
:return: None
"""
try:
db.session.delete(self)
db.session.commit()
except UnmappedInstanceError as e:
db.session.rollback()
raise e
def add_host(self, host):
"""Appends into the laboratory a new host
:param host:
:return: None
"""
if not self.has_host(host):
self.hosts.append(host)
def remove_host(self, host):
"""Removes from database an existent host
:param host:
:return: None
"""
if self.has_host(host):
self.hosts.remove(host)
def has_host(self, host):
"""Checks if the host is appended in the laboratory
:param host:
:return: True or False
"""
return True if host in self.hosts else False
@staticmethod
def get(lab_uuid=None):
"""Gets a list with all laboratories if None lab_uuid is passed. If lab_uuid is present, it returns the
laboratory defined by lab_uuid
:param lab_uuid: UUID of the laboratory
:return:
"""
try:
return Lab.query.get(lab_uuid) if lab_uuid else Lab.query.all()
except Exception as e:
raise e
def update(self, data):
if 'code' in data and data['code'] != "":
self.code = data['code'].upper()
if 'description' in data and data['description'] != "":
self.description = data['description']
if 'start_ip_range' in data and data['start_ip_range'] != "":
self.start_ip_range = data['start_ip_range']
if 'end_ip_range' in data and data['end_ip_range'] != "":
self.end_ip_range = data['end_ip_range']
if 'hosts' in data:
if 'vcpus' in data['hosts'] and data['hosts']['vcpus'] != "":
self.hosts_vcpus = data['hosts']['vcpus']
if 'memory' in data['hosts'] and data['hosts']['memory'] != "":
self.hosts_memory = data['hosts']['memory']
if 'disk' in data['hosts'] and data['hosts']['disk'] != "":
self.hosts_disk = data['hosts']['disk']
try:
db.session.commit()
except Exception as e:
db.session.rollback()
raise e
@staticmethod
def delete(lab):
try:
db.session.delete(lab)
db.session.commit()
except UnmappedInstanceError as e:
db.session.rollback()
raise e
def to_dict(self):
return dict(uuid=str(self.uuid),
code=self.code,
description=self.description,
start_ip_range=self.start_ip_range.compressed,
end_ip_range=self.end_ip_range.compressed,
hosts=dict(total=self.hosts.__len__(),
vcpus=self.hosts_vcpus,
memory=self.hosts_memory,
disk=self.hosts_disk))
class ImportExportMixin:
uuid = sa.Column(UUIDType(binary=True),
primary_key=False,
unique=True,
default=uuid.uuid4)
export_parent: Optional[str] = None
# The name of the attribute
# with the SQL Alchemy back reference
export_children: List[str] = []
# List of (str) names of attributes
# with the SQL Alchemy forward references
export_fields: List[str] = []
# The names of the attributes
# that are available for import and export
__mapper__: Mapper
@classmethod
def _parent_foreign_key_mappings(cls) -> Dict[str, str]:
"""Get a mapping of foreign name to the local name of foreign keys"""
parent_rel = cls.__mapper__.relationships.get(cls.export_parent)
if parent_rel:
return {l.name: r.name for (l, r) in parent_rel.local_remote_pairs}
return {}
@classmethod
def _unique_constrains(cls) -> List[Set[str]]:
"""Get all (single column and multi column) unique constraints"""
unique = [
{c.name
for c in u.columns} for u in cls.__table_args__ # type: ignore
if isinstance(u, UniqueConstraint)
]
unique.extend({c.name} for c in cls.__table__.columns
if c.unique # type: ignore
)
return unique
@classmethod
def parent_foreign_key_mappings(cls) -> Dict[str, str]:
"""Get a mapping of foreign name to the local name of foreign keys"""
parent_rel = cls.__mapper__.relationships.get(cls.export_parent)
if parent_rel:
return {l.name: r.name for (l, r) in parent_rel.local_remote_pairs}
return {}
@classmethod
def export_schema(cls,
recursive: bool = True,
include_parent_ref: bool = False) -> Dict[str, Any]:
"""Export schema as a dictionary"""
parent_excludes = set()
if not include_parent_ref:
parent_ref = cls.__mapper__.relationships.get(cls.export_parent)
if parent_ref:
parent_excludes = {
column.name
for column in parent_ref.local_columns
}
def formatter(column: sa.Column) -> str:
return ("{0} Default ({1})".format(str(column.type),
column.default.arg)
if column.default else str(column.type))
schema: Dict[str, Any] = {
column.name: formatter(column)
for column in cls.__table__.columns # type: ignore
if (column.name in cls.export_fields
and column.name not in parent_excludes)
}
if recursive:
for column in cls.export_children:
child_class = cls.__mapper__.relationships[
column].argument.class_
schema[column] = [
child_class.export_schema(
recursive=recursive,
include_parent_ref=include_parent_ref)
]
return schema
@classmethod
def import_from_dict(
# pylint: disable=too-many-arguments,too-many-branches,too-many-locals
cls,
session: Session,
dict_rep: Dict[Any, Any],
parent: Optional[Any] = None,
recursive: bool = True,
sync: Optional[List[str]] = None,
) -> Any:
"""Import obj from a dictionary"""
if sync is None:
sync = []
parent_refs = cls.parent_foreign_key_mappings()
export_fields = set(cls.export_fields) | set(
parent_refs.keys()) | {"uuid"}
new_children = {
c: dict_rep[c]
for c in cls.export_children if c in dict_rep
}
unique_constrains = cls._unique_constrains()
filters = [] # Using these filters to check if obj already exists
# Remove fields that should not get imported
for k in list(dict_rep):
if k not in export_fields:
del dict_rep[k]
if not parent:
if cls.export_parent:
for prnt in parent_refs.keys():
if prnt not in dict_rep:
raise RuntimeError("{0}: Missing field {1}".format(
cls.__name__, prnt))
else:
# Set foreign keys to parent obj
for k, v in parent_refs.items():
dict_rep[k] = getattr(parent, v)
# Add filter for parent obj
filters.extend(
[getattr(cls, k) == dict_rep.get(k) for k in parent_refs.keys()])
# Add filter for unique constraints
ucs = [
and_(*[
getattr(cls, k) == dict_rep.get(k) for k in cs
if dict_rep.get(k) is not None
]) for cs in unique_constrains
]
filters.append(or_(*ucs))
# Check if object already exists in DB, break if more than one is found
try:
obj_query = session.query(cls).filter(and_(*filters))
obj = obj_query.one_or_none()
except MultipleResultsFound as ex:
logger.error(
"Error importing %s \n %s \n %s",
cls.__name__,
str(obj_query),
yaml.safe_dump(dict_rep),
)
raise ex
if not obj:
is_new_obj = True
# Create new DB object
obj = cls(**dict_rep) # type: ignore
logger.info("Importing new %s %s", obj.__tablename__, str(obj))
if cls.export_parent and parent:
setattr(obj, cls.export_parent, parent)
session.add(obj)
else:
is_new_obj = False
logger.info("Updating %s %s", obj.__tablename__, str(obj))
# Update columns
for k, v in dict_rep.items():
setattr(obj, k, v)
# Recursively create children
if recursive:
for child in cls.export_children:
child_class = cls.__mapper__.relationships[
child].argument.class_
added = []
for c_obj in new_children.get(child, []):
added.append(
child_class.import_from_dict(session=session,
dict_rep=c_obj,
parent=obj,
sync=sync))
# If children should get synced, delete the ones that did not
# get updated.
if child in sync and not is_new_obj:
back_refs = child_class.parent_foreign_key_mappings()
delete_filters = [
getattr(child_class,
k) == getattr(obj, back_refs.get(k))
for k in back_refs.keys()
]
to_delete = set(
session.query(child_class).filter(
and_(*delete_filters))).difference(set(added))
for o in to_delete:
logger.info("Deleting %s %s", child, str(obj))
session.delete(o)
return obj
def export_to_dict(
self,
recursive: bool = True,
include_parent_ref: bool = False,
include_defaults: bool = False,
export_uuids: bool = False,
) -> Dict[Any, Any]:
"""Export obj to dictionary"""
export_fields = set(self.export_fields)
if export_uuids:
export_fields.add("uuid")
if "id" in export_fields:
export_fields.remove("id")
cls = self.__class__
parent_excludes = set()
if recursive and not include_parent_ref:
parent_ref = cls.__mapper__.relationships.get(cls.export_parent)
if parent_ref:
parent_excludes = {c.name for c in parent_ref.local_columns}
dict_rep = {
c.name: getattr(self, c.name)
for c in cls.__table__.columns # type: ignore
if (c.name in export_fields and c.name not in parent_excludes and (
include_defaults or (getattr(self, c.name) is not None and (
not c.default or getattr(self, c.name) != c.default.arg))))
}
# sort according to export_fields using DSU (decorate, sort, undecorate)
order = {field: i for i, field in enumerate(self.export_fields)}
decorated_keys = [(order.get(k, len(order)), k) for k in dict_rep]
decorated_keys.sort()
dict_rep = {k: dict_rep[k] for _, k in decorated_keys}
if recursive:
for cld in self.export_children:
# sorting to make lists of children stable
dict_rep[cld] = sorted(
[
child.export_to_dict(
recursive=recursive,
include_parent_ref=include_parent_ref,
include_defaults=include_defaults,
) for child in getattr(self, cld)
],
key=lambda k: sorted(str(k.items())),
)
return convert_uuids(dict_rep)
def override(self, obj: Any) -> None:
"""Overrides the plain fields of the dashboard."""
for field in obj.__class__.export_fields:
setattr(self, field, getattr(obj, field))
def copy(self) -> Any:
"""Creates a copy of the dashboard without relationships."""
new_obj = self.__class__()
new_obj.override(self)
return new_obj
def alter_params(self, **kwargs: Any) -> None:
params = self.params_dict
params.update(kwargs)
self.params = json.dumps(params)
def remove_params(self, param_to_remove: str) -> None:
params = self.params_dict
params.pop(param_to_remove, None)
self.params = json.dumps(params)
def reset_ownership(self) -> None:
""" object will belong to the user the current user """
# make sure the object doesn't have relations to a user
# it will be filled by appbuilder on save
self.created_by = None
self.changed_by = None
# flask global context might not exist (in cli or tests for example)
self.owners = []
if g and hasattr(g, "user"):
self.owners = [g.user]
@property
def params_dict(self) -> Dict[Any, Any]:
return json_to_dict(self.params)
@property
def template_params_dict(self) -> Dict[Any, Any]:
return json_to_dict(self.template_params) # type: ignore
class Argument(Execution_Base, Validatable):
__tablename__ = 'argument'
id = Column(Integer, primary_key=True, autoincrement=True)
action_id = Column(UUIDType(binary=False),
ForeignKey('action.id', ondelete='CASCADE'))
action_device_id = Column(UUIDType(binary=False),
ForeignKey('action.id', ondelete='CASCADE'))
condition_id = Column(UUIDType(binary=False),
ForeignKey('condition.id', ondelete='CASCADE'))
transform_id = Column(UUIDType(binary=False),
ForeignKey('transform.id', ondelete='CASCADE'))
name = Column(String(255), nullable=False)
value = Column(JSONType)
reference = Column(UUIDType(binary=False))
selection = Column(ScalarListType())
errors = Column(ScalarListType())
def __init__(self, name, value=None, reference=None, selection=None):
"""Initializes an Argument object.
Args:
name (str): The name of the Argument.
value (any, optional): The value of the Argument. Defaults to None. Value or reference must be included.
reference (int, optional): The ID of the Action from which to grab the result. Defaults to None.
If value is not provided, then reference must be included.
selection (list, optional): A list of fields from which to dereference the Action result. Defaults
to None. Must be used in conjunction with reference.
"""
self.name = name
self.value = value
self._is_reference = True if value is None else False
self.reference = reference
self.selection = selection
self.validate()
@orm.reconstructor
def init_on_load(self):
"""Loads all necessary fields upon Argument being loaded from database"""
self._is_reference = True if self.value is None else False
def validate(self):
"""Validates the object"""
self.errors = []
if self.value is None and not self.reference:
message = 'Input {} must have either value or reference. Input has neither'.format(
self.name)
logger.error(message)
self.errors = [message]
elif self.value is not None and self.reference:
message = 'Input {} must have either value or reference. Input has both. Using "value"'.format(
self.name)
logger.warning(message)
self.reference = None
def update_value_reference(self, value, reference):
"""Helper function to ensure that either reference or value is selected and the other is None
Args:
value (any): The value to set. Can be None
reference (int): The reference to set. Can be none
"""
if value is not None and (self.value != value or self.reference):
self.value = value
self.reference = None
self.selection = []
elif reference:
self.reference = reference
self.value = None
self._is_reference = True
@property
def is_ref(self):
"""Returns whether the reference field is being used, or the value field.
Returns:
(bool): True if the reference field is being used, False if otherwise.
"""
return self._is_reference
def get_value(self, accumulator):
"""Returns the value associated with this Argument, either by returning Argument.value, or using the
accumulator to dereference the associated Action output.
Args:
accumulator (dict): The accumulated output from previous Actions.
Returns:
(any): The value associated with this Argument.
"""
if self.value is not None:
return self.value
if accumulator:
action_output = self._get_action_from_reference(accumulator)
if not self.selection:
return action_output
return self._select(action_output)
else:
return self.reference
def _get_action_from_reference(self, accumulator):
try:
return accumulator[self.reference]
except KeyError:
message = ('Referenced action {} '
'has not been executed'.format(self.reference))
logger.info(message)
raise InvalidArgument(message)
def _select(self, input_):
try:
for selection in self.selection:
input_ = Argument._get_next_selection(input_, selection)
return input_
except (KeyError, ValueError, IndexError):
raise InvalidArgument(
'Selector {0} is invalid for reference {1}'.format(
self.selection, self.reference))
@staticmethod
def _get_next_selection(input_, selection):
if isinstance(input_, dict):
return input_[selection]
elif isinstance(input_, list):
return input_[int(selection)]
else:
raise ValueError
@classmethod
def create_device_argument(cls,
value=None,
reference=None,
selection=None):
return cls(name='__device__',
value=value,
reference=reference,
selection=selection)
def __eq__(self, other):
return self.name == other.name and self.value == other.value and self.reference == other.reference and \
self.selection == other.selection and self._is_reference == other.is_ref
def __hash__(self):
return hash(self.id)
class Condition(ExecutionElement, executiondb.Device_Base):
__tablename__ = 'condition'
conditional_expression_id = Column(UUIDType(binary=False),
ForeignKey('conditional_expression.id'))
app_name = Column(String(80), nullable=False)
action_name = Column(String(80), nullable=False)
is_negated = Column(Boolean, default=False)
arguments = relationship('Argument', cascade='all, delete, delete-orphan')
transforms = relationship('Transform', cascade='all, delete-orphan')
def __init__(self,
app_name,
action_name,
id=None,
is_negated=False,
arguments=None,
transforms=None):
"""Initializes a new Condition object.
Args:
app_name (str): The name of the app which contains this condition
action_name (str): The action name for the Condition. Defaults to an empty string.
id (str|UUID, optional): Optional UUID to pass into the Condition. Must be UUID object or valid UUID string.
Defaults to None.
is_negated (bool, optional): Should the result of the condition be inverted? Defaults to False.
arguments (list[Argument], optional): Dictionary of Argument keys to Argument values.
This dictionary will be converted to a dictionary of str:Argument. Defaults to None.
transforms(list[Transform], optional): A list of Transform objects for the Condition object.
Defaults to None.
"""
ExecutionElement.__init__(self, id)
self.app_name = app_name
self.action_name = action_name
self.is_negated = is_negated
self.arguments = []
if arguments:
self.arguments = arguments
self.transforms = []
if transforms:
self.transforms = transforms
self._data_param_name = None
self._run = None
self._api = None
self._condition_executable = None
self.validate()
@orm.reconstructor
def init_on_load(self):
"""Loads all necessary fields upon Condition being loaded from database"""
self._data_param_name, self._run, self._api = get_condition_api(
self.app_name, self.action_name)
self._condition_executable = get_condition(self.app_name, self._run)
def validate(self):
errors = {}
try:
self._data_param_name, self._run, self._api = get_condition_api(
self.app_name, self.action_name)
self._condition_executable = get_condition(self.app_name,
self._run)
tmp_api = split_api_params(self._api, self._data_param_name)
validate_condition_parameters(tmp_api, self.arguments,
self.action_name)
except UnknownApp:
errors['executable'] = 'Unknown app {}'.format(self.app_name)
except UnknownCondition:
errors['executable'] = 'Unknown condition {}'.format(
self.action_name)
except InvalidArgument as e:
errors['arguments'] = e.errors
if errors:
raise InvalidExecutionElement(self.id,
self.action_name,
'Invalid condition {}'.format(
self.id or self.action_name),
errors=[errors])
def execute(self, data_in, accumulator):
"""Executes the Condition object, determining if the Condition evaluates to True or False.
Args:
data_in (): The input to the Transform objects associated with this Condition.
accumulator (dict): The accumulated data from previous Actions.
Returns:
True if the Condition evaluated to True, False otherwise
"""
data = data_in
for transform in self.transforms:
data = transform.execute(data, accumulator)
try:
arguments = self.__update_arguments_with_data(data)
args = validate_condition_parameters(self._api,
arguments,
self.action_name,
accumulator=accumulator)
logger.debug('Arguments passed to condition {} are valid'.format(
self.id))
ret = self._condition_executable(**args)
WalkoffEvent.CommonWorkflowSignal.send(
self, event=WalkoffEvent.ConditionSuccess)
if self.is_negated:
return not ret
else:
return ret
except InvalidArgument as e:
logger.error(
'Condition {0} has invalid input {1} which was converted to {2}. Error: {3}. '
'Returning False'.format(self.action_name, data_in, data,
format_exception_message(e)))
WalkoffEvent.CommonWorkflowSignal.send(
self, event=WalkoffEvent.ConditionError)
raise
except Exception as e:
logger.error('Error encountered executing '
'condition {0} with arguments {1} and value {2}: '
'Error {3}. Returning False'.format(
self.action_name, arguments, data,
format_exception_message(e)))
WalkoffEvent.CommonWorkflowSignal.send(
self, event=WalkoffEvent.ConditionError)
raise
def __update_arguments_with_data(self, data):
arguments = []
for argument in self.arguments:
if argument.name != self._data_param_name:
arguments.append(argument)
arguments.append(Argument(self._data_param_name, value=data))
return arguments
class Notification(db.Model, BaseMixin, SerializerMixin):
"""
A notification is stored each time a comment is posted.
"""
read = db.Column(db.Boolean, nullable=False, default=False)
change = db.Column(db.Boolean, nullable=False, default=False)
type = db.Column(ChoiceType(TYPES))
person_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("person.id"),
nullable=False,
index=True,
)
author_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("person.id"),
nullable=False,
index=True,
)
comment_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("comment.id"),
nullable=True,
index=True,
)
task_id = db.Column(
UUIDType(binary=False),
db.ForeignKey("task.id"),
nullable=False,
index=True,
)
__table_args__ = (db.UniqueConstraint(
"person_id",
"author_id",
"comment_id",
"type",
name="notification_uc",
), )
def serialize(self, obj_type=None, relations=False):
attrs = inspect(self).attrs.keys()
obj_dict = {
attr: serialize_value(getattr(self, attr))
for attr in attrs
}
obj_dict["notification_type"] = obj_dict["type"]
obj_dict["type"] = obj_type or type(self).__name__
return obj_dict
@classmethod
def create_from_import(cls, data):
notification_type = ""
if "notification_type" in data:
notification_type = data.get("notification_type", "")
del data["notification_type"]
data["type"] = notification_type
previous_data = cls.get(data["id"])
if previous_data is None:
return (cls.create(**data), False)
else:
previous_data.update(data)
return (previous_data, True)
