class CheckerRecord(db.Model):
"""Connect checks with their executions on records."""
__tablename__ = 'checker_record'
rec_id = db.Column(
db.MediumInteger(8, unsigned=True),
db.ForeignKey(RecordMetadata.id),
primary_key=True,
nullable=False,
autoincrement=True,
)
record = db.relationship(
RecordMetadata,
backref=backref("checker_record", cascade="all, delete-orphan"),
doc="The record associated with a task.",
)
rule_name = db.Column(db.String(127),
db.ForeignKey('checker_rule.name'),
nullable=False,
index=True,
primary_key=True,
doc="Name of the task in this associaton.")
last_run_version_id = db.Column(
db.Integer,
nullable=False,
doc="Last checked version ID of associated record.",
)
class FieldTag(db.Model):
"""Represent a FieldTag record."""
__tablename__ = 'field_tag'
id_field = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey('field.id'),
nullable=False,
primary_key=True)
id_tag = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey('tag.id'),
nullable=False,
primary_key=True)
score = db.Column(db.TinyInteger(4, unsigned=True),
nullable=False,
server_default='0')
tag = db.relationship(Tag, backref='fields', order_by=score)
field = db.relationship(Field, backref='tags', order_by=score)
def __init__(self, score=None, tup=None, *args, **kwargs):
"""Init."""
if score is not None:
self.score = score
if tup is not None:
self.tag = Tag(tup)
super(FieldTag, self).__init__(*args, **kwargs)
@property
def as_tag(self):
"""Return Tag record directly."""
return self.tag
class UserQuery(db.Model):
"""Represent a UserQuery record."""
__tablename__ = 'user_query'
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
primary_key=True,
server_default='0')
id_query = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(WebQuery.id),
primary_key=True,
index=True,
server_default='0')
hostname = db.Column(db.String(50),
nullable=True,
server_default='unknown host')
date = db.Column(db.DateTime, nullable=True, default=datetime.datetime.now)
webquery = db.relationship(WebQuery, backref='executions')
@classmethod
def log(cls, urlargs=None, id_user=None):
"""Log."""
id_user = id_user if not None else current_user.get_id()
urlargs = urlargs or request.query_string
if id_user < 0:
return
webquery = WebQuery.query.filter_by(urlargs=urlargs).first()
if webquery is None:
webquery = WebQuery(urlargs=urlargs)
db.session.add(
cls(id_user=id_user, hostname=request.host, webquery=webquery))
db.session.commit()
class KnwKBDDEF(db.Model):
"""Represent a KnwKBDDEF record."""
__tablename__ = 'knwKBDDEF'
id_knwKB = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(KnwKB.id),
nullable=False,
primary_key=True)
id_collection = db.Column(db.MediumInteger(unsigned=True),
db.ForeignKey(Collection.id),
nullable=True)
output_tag = db.Column(db.Text, nullable=True)
search_expression = db.Column(db.Text, nullable=True)
kb = db.relationship(KnwKB,
backref=db.backref('kbdefs',
uselist=False,
cascade="all, delete-orphan"),
single_parent=True)
collection = db.relationship(Collection, backref=db.backref('kbdefs'))
def to_dict(self):
"""Return a dict representation of KnwKBDDEF."""
return {
'field': self.output_tag,
'expression': self.search_expression,
'coll_id': self.id_collection,
'collection': self.collection.name if self.collection else None
}
class WtgTAGRecord(db.Model, Serializable):
"""Connection between Tag and Record."""
__tablename__ = 'wtgTAG_bibrec'
__public__ = set(['id_tag', 'id_bibrec', 'date_added'])
# tagTAG.id
id_tag = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(WtgTAG.id),
nullable=False,
primary_key=True)
# Record.id
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Record.id),
nullable=False,
primary_key=True)
# Annotation
annotation = db.Column(
db.Text(convert_unicode=True),
default='')
# Creation date
date_added = db.Column(db.DateTime,
default=datetime.now)
# Relationships
tag = db.relationship(WtgTAG,
backref=db.backref('records_association',
cascade='all'))
tag_query = db.relationship(WtgTAG,
backref=db.backref('records_association_query',
cascade='all',
lazy='dynamic'))
bibrec = db.relationship(Record,
backref=db.backref('tags_association',
cascade='all'))
bibrec_query = db.relationship(Record,
backref=db.backref('tags_association_query',
cascade='all',
lazy='dynamic'))
def __init__(self, bibrec=None, **kwargs):
"""TODO."""
super(WtgTAGRecord, self).__init__(**kwargs)
if bibrec is not None:
self.bibrec = bibrec
class CmtCOLLAPSED(db.Model):
"""Represents a CmtCOLLAPSED record."""
__tablename__ = 'cmtCOLLAPSED'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
primary_key=True)
id_cmtRECORDCOMMENT = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(CmtRECORDCOMMENT.id),
primary_key=True)
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
primary_key=True)
class UserAccROLE(db.Model):
"""Represent an user role relationship."""
__tablename__ = 'user_accROLE'
id_user = db.Column(db.Integer(15, unsigned=True), db.ForeignKey(User.id),
nullable=False, primary_key=True)
id_accROLE = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(AccROLE.id), nullable=False,
primary_key=True)
expiration = db.Column(db.DateTime, nullable=False,
server_default='9999-12-31 23:59:59')
user = db.relationship(User, backref='roles')
role = db.relationship(AccROLE, backref='users')
class PidLog(db.Model):
"""Audit log of actions happening to persistent identifiers.
This model is primarily used through PersistentIdentifier.log and rarely
created manually.
"""
__tablename__ = 'pidLOG'
__table_args__ = (
db.Index('idx_action', 'action'),
)
id = db.Column(db.Integer(15, unsigned=True), primary_key=True)
"""Id of persistent identifier entry."""
id_pid = db.Column(
db.Integer(15, unsigned=True), db.ForeignKey(PersistentIdentifier.id),
nullable=True,
)
"""PID."""
timestamp = db.Column(db.DateTime(), nullable=False, default=datetime.now)
"""Creation datetime of entry."""
action = db.Column(db.String(10), nullable=False)
"""Action identifier."""
message = db.Column(db.Text(), nullable=False)
"""Log message."""
# Relationship
pid = db.relationship("PersistentIdentifier", backref="logs")
class CmtSUBSCRIPTION(db.Model):
"""Represents a CmtSUBSCRIPTION record."""
__tablename__ = 'cmtSUBSCRIPTION'
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False,
primary_key=True)
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=False,
primary_key=True)
creation_time = db.Column(db.DateTime,
nullable=False,
server_default='1900-01-01 00:00:00')
bibrec = db.relationship(Bibrec)
user = db.relationship(User, backref='comment_subscriptions')
class Fieldname(db.Model):
"""Represent a Fieldname record."""
__tablename__ = 'fieldname'
id_field = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(Field.id),
primary_key=True)
ln = db.Column(db.Char(5), primary_key=True, server_default='')
type = db.Column(db.Char(3), primary_key=True, server_default='sn')
value = db.Column(db.String(255), nullable=False)
field = db.relationship(Field, backref='names')
class AccAuthorization(db.Model):
"""Represent an authorization."""
__tablename__ = 'accROLE_accACTION_accARGUMENT'
id = db.Column(db.Integer(15, unsigned=True), primary_key=True,
autoincrement=True)
id_accROLE = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(AccROLE.id), nullable=True,
index=True)
id_accACTION = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(AccACTION.id), nullable=True,
index=True)
_id_accARGUMENT = db.Column(db.Integer(15), nullable=True,
name="id_accARGUMENT", index=True)
argumentlistid = db.Column(db.MediumInteger(8), nullable=True)
role = db.relationship(AccROLE, backref='authorizations')
action = db.relationship(AccACTION, backref='authorizations')
argument = db.relationship(
AccARGUMENT, backref='authorizations',
primaryjoin=db.and_(
AccARGUMENT.id == _id_accARGUMENT,
_id_accARGUMENT != -1,
_id_accARGUMENT is not None
),
foreign_keys=_id_accARGUMENT,
uselist=False,
cascade="all, delete",
)
@db.hybrid_property
def id_accARGUMENT(self):
"""get id_accARGUMENT."""
return self._id_accARGUMENT
@id_accARGUMENT.setter
def id_accARGUMENT(self, value):
"""set id_accARGUMENT."""
self._id_accARGUMENT = value or None
class CmtACTIONHISTORY(db.Model):
"""Represents a CmtACTIONHISTORY record."""
__tablename__ = 'cmtACTIONHISTORY'
id_cmtRECORDCOMMENT = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(CmtRECORDCOMMENT.id),
nullable=True,
primary_key=True)
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=True,
primary_key=True)
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=True,
primary_key=True)
client_host = db.Column(db.Integer(10, unsigned=True), nullable=True)
action_time = db.Column(db.DateTime,
nullable=False,
server_default='1900-01-01 00:00:00')
action_code = db.Column(db.Char(1), nullable=False, index=True)
recordcomment = db.relationship(CmtRECORDCOMMENT, backref='actionhistory')
bibrec = db.relationship(Bibrec)
user = db.relationship(User)
class UserEXT(db.Model):
"""Represent a UserEXT record."""
__tablename__ = 'userEXT'
id = db.Column(db.String(255), primary_key=True, nullable=False)
method = db.Column(db.String(50), primary_key=True, nullable=False)
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=False)
user = db.relationship(User, backref="external_identifiers")
__table_args__ = (db.Index('userext_id_user_method',
id_user,
method,
unique=True), db.Model.__table_args__)
class RecordMetadata(db.Model):
"""Represent a json record inside the SQL database."""
__tablename__ = 'record_json'
id = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Record.id),
primary_key=True,
nullable=False,
autoincrement=True)
version_id = db.Column(db.Integer, nullable=False)
json = db.Column(db.JSON, nullable=False)
record = db.relationship(Record, backref='record_json')
__mapper_args__ = {"version_id_col": version_id}
class CheckerReporter(db.Model):
"""Represent reporters associated with a task.
..note::
These entries are currently not meant to be associated with multiple
tasks, because it is assumed that they may be deleted without affecting
more than one tasks.
"""
__tablename__ = 'checker_reporter'
plugin = db.Column(db.String(127),
primary_key=True,
doc="Check associated with this reporter."
"")
rule_name = db.Column(db.String(127),
db.ForeignKey('checker_rule.name',
onupdate="CASCADE",
ondelete="CASCADE"),
index=True,
nullable=False,
primary_key=True,
doc="Task associated with this reporter."
"")
arguments = db.Column(
JsonEncodedDict(1023),
default={},
doc="Arguments to be passed to this reporter.",
)
@db.hybrid_property
def module(self):
"""Python module of the associated check."""
return reporters_files[self.plugin]
class WtgTAG(db.Model, Serializable):
"""A Tag."""
__tablename__ = 'wtgTAG'
__public__ = set(['id', 'name', 'id_owner'])
#
# Access Rights
#
ACCESS_NAMES = {
0: 'Nothing',
10: 'View',
20: 'Add',
30: 'Add and remove',
40: 'Manage',
}
ACCESS_LEVELS = \
dict((v, k) for (k, v) in iteritems(ACCESS_NAMES))
ACCESS_RIGHTS = {
0: [],
10: ['view'],
20: ['view', 'add'],
30: ['view', 'add', 'remove'],
40: ['view', 'add', 'remove', 'edit'],
}
ACCESS_OWNER_DEFAULT = ACCESS_LEVELS['Manage']
ACCESS_GROUP_DEFAULT = ACCESS_LEVELS['View']
# Primary key
id = db.Column(db.Integer(15, unsigned=True),
primary_key=True,
nullable=False,
autoincrement=True)
# Name
name = db.Column(db.String(255),
nullable=False,
server_default='',
index=True)
# Owner
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=True)
# Access rights of owner
user_access_rights = db.Column(db.Integer(2, unsigned=True),
nullable=False,
default=ACCESS_OWNER_DEFAULT)
# Group
# equal to NULL for private tags
id_usergroup = db.Column(
db.Integer(15, unsigned=True),
db.ForeignKey(Group.id),
nullable=True)
# Group access rights
group_access_rights = db.Column(
db.Integer(2, unsigned=True),
nullable=False,
default=ACCESS_GROUP_DEFAULT)
# Access rights of everyone
public_access_rights = db.Column(db.Integer(2, unsigned=True),
nullable=False,
default=ACCESS_LEVELS['Nothing'])
# Visibility in document description
show_in_description = db.Column(db.Boolean,
nullable=False,
default=True)
# Relationships
user = db.relationship(User,
backref=db.backref('tags', cascade='all'))
user_query = db.relationship(User,
backref=db.backref('tags_query',
cascade='all',
lazy='dynamic'))
usergroup = db.relationship(
Group,
backref=db.backref('tags', cascade='all'))
# association proxy of "user_keywords" collection
# to "keyword" attribute
records = association_proxy('records_association', 'bibrec')
# Calculated fields
@db.hybrid_property
def record_count(self):
"""TODO."""
return self.records_association_query.count()
@record_count.expression
def record_count(cls):
"""TODO."""
return db.select([db.func.count(WtgTAGRecord.id_bibrec)]) \
.where(WtgTAGRecord.id_tag == cls.id) \
.label('record_count')
@db.validates('user_access_rights')
@db.validates('group_access_rights')
@db.validates('public_access_rights')
def validate_user_access_rights(self, key, value):
"""Check if the value is among defined levels."""
assert value in WtgTAG.ACCESS_NAMES
return value
class Client(db.Model):
"""A client is the app which want to use the resource of a user.
It is suggested that the client is registered by a user on your site, but
it is not required.
The client should contain at least these information:
client_id: A random string
client_secret: A random string
client_type: A string represents if it is confidential
redirect_uris: A list of redirect uris
default_redirect_uri: One of the redirect uris
default_scopes: Default scopes of the client
But it could be better, if you implemented:
allowed_grant_types: A list of grant types
allowed_response_types: A list of response types
validate_scopes: A function to validate scopes
"""
__tablename__ = 'oauth2CLIENT'
name = db.Column(
db.String(40),
info=dict(
label=_('Name'),
description=_('Name of application (displayed to users).'),
validators=[validators.DataRequired()]
)
)
"""Human readable name of the application."""
description = db.Column(
db.Text(),
default=u'',
info=dict(
label=_('Description'),
description=_('Optional. Description of the application'
' (displayed to users).'),
)
)
"""Human readable description."""
website = db.Column(
URLType(),
info=dict(
label=_('Website URL'),
description=_('URL of your application (displayed to users).'),
),
default=u'',
)
user_id = db.Column(db.ForeignKey('user.id'), nullable=True)
"""Creator of the client application."""
client_id = db.Column(db.String(255), primary_key=True)
"""Client application ID."""
client_secret = db.Column(
db.String(255), unique=True, index=True, nullable=False
)
"""Client application secret."""
is_confidential = db.Column(db.Boolean, default=True)
"""Determine if client application is public or not."""
is_internal = db.Column(db.Boolean, default=False)
"""Determins if client application is an internal application."""
_redirect_uris = db.Column(db.Text)
"""A newline-separated list of redirect URIs. First is the default URI."""
_default_scopes = db.Column(db.Text)
"""A space-separated list of default scopes of the client.
The value of the scope parameter is expressed as a list of space-delimited,
case-sensitive strings.
"""
user = db.relationship(
User,
backref=db.backref(
"oauth2clients",
cascade="all, delete-orphan",
)
)
"""Relationship to user."""
@property
def allowed_grant_types(self):
"""Return allowed grant types."""
return current_app.config['OAUTH2_ALLOWED_GRANT_TYPES']
@property
def allowed_response_types(self):
"""Return allowed response types."""
return current_app.config['OAUTH2_ALLOWED_RESPONSE_TYPES']
# def validate_scopes(self, scopes):
# return self._validate_scopes
@property
def client_type(self):
"""Return client type."""
if self.is_confidential:
return 'confidential'
return 'public'
@property
def redirect_uris(self):
"""Return redirect uris."""
if self._redirect_uris:
return self._redirect_uris.splitlines()
return []
@redirect_uris.setter
def redirect_uris(self, value):
"""Validate and store redirect URIs for client."""
if isinstance(value, six.text_type):
value = value.split("\n")
value = [v.strip() for v in value]
for v in value:
validate_redirect_uri(v)
self._redirect_uris = "\n".join(value) or ""
@property
def default_redirect_uri(self):
"""Return default redirect uri."""
try:
return self.redirect_uris[0]
except IndexError:
pass
@property
def default_scopes(self):
"""List of default scopes for client."""
if self._default_scopes:
return self._default_scopes.split(" ")
return []
@default_scopes.setter
def default_scopes(self, scopes):
"""Set default scopes for client."""
validate_scopes(scopes)
self._default_scopes = " ".join(set(scopes)) if scopes else ""
def validate_scopes(self, scopes):
"""Validate if client is allowed to access scopes."""
try:
validate_scopes(scopes)
return True
except ScopeDoesNotExists:
return False
def gen_salt(self):
"""Generate salt."""
self.reset_client_id()
self.reset_client_secret()
def reset_client_id(self):
"""Reset client id."""
self.client_id = gen_salt(
current_app.config.get('OAUTH2_CLIENT_ID_SALT_LEN')
)
def reset_client_secret(self):
"""Reset client secret."""
self.client_secret = gen_salt(
current_app.config.get('OAUTH2_CLIENT_SECRET_SALT_LEN')
)
class KnwKBRVAL(db.Model):
"""Represent a KnwKBRVAL record."""
__tablename__ = 'knwKBRVAL'
m_key = db.Column(db.String(255),
nullable=False,
primary_key=True,
index=True)
m_value = db.Column(db.Text().with_variant(mysql.TEXT(30), 'mysql'),
nullable=False)
id_knwKB = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(KnwKB.id),
nullable=False,
server_default='0',
primary_key=True)
kb = db.relationship(
KnwKB,
backref=db.backref(
'kbrvals',
cascade="all, delete-orphan",
collection_class=attribute_mapped_collection("m_key")))
@staticmethod
def query_kb_mappings(kbid, sortby="to", key="", value="", match_type="s"):
"""Return a list of all mappings from the given kb, ordered by key.
If key given, give only those with left side (mapFrom) = key.
If value given, give only those with right side (mapTo) = value.
:param kb_name: knowledge base name. if "", return all
:param sortby: the sorting criteria ('from' or 'to')
:param key: return only entries where key matches this
:param value: return only entries where value matches this
:param match_type: s=substring, e=exact, sw=startswith
"""
# query
query = KnwKBRVAL.query.filter(KnwKBRVAL.id_knwKB == kbid)
# filter
if len(key) > 0:
if match_type == "s":
key = "%" + key + "%"
elif match_type == "sw":
key = key + "%"
else:
key = '%'
if len(value) > 0:
if match_type == "s":
value = "%" + value + "%"
elif match_type == "sw":
value = value + "%"
else:
value = '%'
query = query.filter(KnwKBRVAL.m_key.like(key),
KnwKBRVAL.m_value.like(value))
# order by
if sortby == "from":
query = query.order_by(KnwKBRVAL.m_key)
else:
query = query.order_by(KnwKBRVAL.m_value)
return query
def to_dict(self):
"""Return a dict representation of KnwKBRVAL."""
# FIXME remove 'id' dependency from invenio modules
return {
'id': self.m_key + "_" + str(self.id_knwKB),
'key': self.m_key,
'value': self.m_value,
'kbid': self.kb.id if self.kb else None,
'kbname': self.kb.name if self.kb else None
}
class CmtRECORDCOMMENT(db.Model):
"""Represents a CmtRECORDCOMMENT record."""
__tablename__ = 'cmtRECORDCOMMENT'
id = db.Column(db.Integer(15, unsigned=True),
nullable=False,
primary_key=True,
autoincrement=True)
id_bibrec = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(Bibrec.id),
nullable=False,
server_default='0')
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=False,
server_default='0')
title = db.Column(db.String(255), nullable=False, server_default='')
body = db.Column(db.Text, nullable=False)
date_creation = db.Column(db.DateTime,
nullable=False,
server_default='1900-01-01 00:00:00')
star_score = db.Column(db.TinyInteger(5, unsigned=True),
nullable=False,
server_default='0')
nb_votes_yes = db.Column(db.Integer(10),
nullable=False,
server_default='0')
nb_votes_total = db.Column(db.Integer(10, unsigned=True),
nullable=False,
server_default='0')
nb_abuse_reports = db.Column(db.Integer(10),
nullable=False,
server_default='0')
status = db.Column(db.Char(2),
nullable=False,
index=True,
server_default='ok')
round_name = db.Column(db.String(255), nullable=False, server_default='')
restriction = db.Column(db.String(50), nullable=False, server_default='')
in_reply_to_id_cmtRECORDCOMMENT = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(id),
nullable=False,
server_default='0')
reply_order_cached_data = db.Column(db.Binary, nullable=True)
bibrec = db.relationship(Bibrec, backref='recordcomments')
user = db.relationship(User, backref='recordcomments')
replies = db.relationship('CmtRECORDCOMMENT',
backref=db.backref('parent',
remote_side=[id],
order_by=date_creation))
@property
def is_deleted(self):
"""Check if is deleted."""
return self.status != 'ok'
def is_collapsed(self, id_user):
"""Return true if the comment is collapsed by user."""
return CmtCOLLAPSED.query.filter(
db.and_(CmtCOLLAPSED.id_bibrec == self.id_bibrec,
CmtCOLLAPSED.id_cmtRECORDCOMMENT == self.id,
CmtCOLLAPSED.id_user == id_user)).count() > 0
@session_manager
def collapse(self, id_user):
"""Collapse comment beloging to user."""
c = CmtCOLLAPSED(id_bibrec=self.id_bibrec,
id_cmtRECORDCOMMENT=self.id,
id_user=id_user)
db.session.add(c)
db.session.commit()
def expand(self, id_user):
"""Expand comment beloging to user."""
CmtCOLLAPSED.query.filter(
db.and_(CmtCOLLAPSED.id_bibrec == self.id_bibrec,
CmtCOLLAPSED.id_cmtRECORDCOMMENT == self.id,
CmtCOLLAPSED.id_user == id_user)).delete(
synchronize_session=False)
__table_args__ = (db.Index('cmtRECORDCOMMENT_reply_order_cached_data',
reply_order_cached_data,
mysql_length=40), db.Model.__table_args__)
@classmethod
def count(cls, *criteria, **filters):
"""Count how many comments."""
return cls.query.filter(*criteria).filter_by(**filters).count()
class CheckerRuleExecution(db.Model):
__tablename__ = 'checker_rule_execution'
uuid = db.Column(
db.String(36),
primary_key=True,
doc="UUID of the execution. Same with that of RedisMaster and logfile.",
)
owner_id = db.Column(
db.Integer(15, unsigned=True),
db.ForeignKey('user.id'),
nullable=False,
default=1,
)
owner = db.relationship(
'User',
doc="User who owns this execution. May be used by reporters.",
)
rule_name = db.Column(
db.String(127),
db.ForeignKey('checker_rule.name'),
nullable=False,
index=True,
doc="Name of the associated task.",
)
_status = db.Column(
ChoiceType(StatusMaster, impl=db.Integer()),
default=StatusMaster.unknown,
)
status_update_date = db.Column(
db.DateTime(),
nullable=False,
server_default='1900-01-01 00:00:00',
doc="Last date the status was updated.",
)
start_date = db.Column(
db.DateTime(),
nullable=False,
server_default='1900-01-01 00:00:00',
doc="Date at which this task was started.",
)
dry_run = db.Column(
db.Boolean,
default=False,
doc=
"Whether this execution is a dry run. Note the `should_*` properties.")
@db.hybrid_property
def should_commit(self):
"""Whether this execution should commit record modifications."""
return not self.dry_run
@db.hybrid_property
def should_report_logs(self):
"""Whether this execution should report logs to the reporters."""
return not self.dry_run
@db.hybrid_property
def should_report_exceptions(self):
"""Whether this execution should report exceptions to the reporters."""
return not self.dry_run
@db.hybrid_property
def master(self):
"""The master object of this execution.
:rtype: `invenio_checker.clients.master.RedisMaster`
"""
return RedisMaster(self.uuid)
@db.hybrid_property
def status(self):
"""The status of the execution.
:rtype: `StatusMaster`
"""
return self._status
@status.setter
@session_manager
def status(self, new_status):
"""Status setter.
:type new_status: `StatusMaster`
"""
self._status = new_status
self.status_update_date = datetime.now()
def read_logs(self):
"""Stream user-friendly structured logs of this execution.
First attempt to stream using `eliot-tree` which provides a text
tree-like structure of the execution given its logs.
If `eliot-tree` fails (which happens when there is an eliot Task
serialization bug in our code, a warning is yielded, followed by the
output of `eliot-prettyprint`.
Therefore, the output of this function is not guaranteed to be
machine-readable.
..note::
This function may be called mid-run.
"""
from glob import glob
import subprocess
from .config import get_eliot_log_file
filenames = glob(get_eliot_log_file(master_id=self.uuid).name + "*")
eliottree_subp = subprocess.Popen(['eliot-tree', '--field-limit', '0'],
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
eliottree_failed = False
with eliottree_subp.stdin:
try:
for filename in filenames:
with open(filename, 'r') as file_:
eliottree_subp.stdin.write(file_.read())
except (IOError, MemoryError):
eliottree_failed = True
with eliottree_subp.stdout:
for line in eliottree_subp.stdout:
yield line
if eliottree_failed or (eliottree_subp.wait() != 0):
# eliot-tree can fail on unfinished logging. We still want output
# for debugging, so we use the less structured eliot-prettyprint
from eliot.prettyprint import pretty_format
from eliot._bytesjson import loads
yield '\n`eliot-tree` failed to format output. ' \
'Retrying with eliot-prettyprint:\n'
for filename in filenames:
yield "{}:\n".format(filename)
with open(filename, 'r') as file_:
for line in file_:
yield pretty_format(loads(line))
class Token(db.Model):
"""A bearer token is the final token that can be used by the client."""
__tablename__ = 'oauth2TOKEN'
id = db.Column(db.Integer, primary_key=True, autoincrement=True)
"""Object ID."""
client_id = db.Column(
db.String(255), db.ForeignKey('oauth2CLIENT.client_id'),
nullable=False,
)
"""Foreign key to client application."""
client = db.relationship(
'Client',
backref=db.backref(
'oauth2tokens',
cascade="all, delete-orphan"
))
"""SQLAlchemy relationship to client application."""
user_id = db.Column(
db.Integer(15, unsigned=True), db.ForeignKey('user.id'), nullable=True
)
"""Foreign key to user."""
user = db.relationship(
User,
backref=db.backref(
"oauth2tokens",
cascade="all, delete-orphan",
)
)
"""SQLAlchemy relationship to user."""
token_type = db.Column(db.String(255), default='bearer')
"""Token type - only bearer is supported at the moment."""
access_token = db.Column(String255EncryptedType(
type_in=db.String(255),
key=secret_key),
unique=True
)
refresh_token = db.Column(String255EncryptedType(
type_in=db.String(255),
key=secret_key,
engine=NoneAesEngine),
unique=True, nullable=True
)
expires = db.Column(db.DateTime, nullable=True)
_scopes = db.Column(db.Text)
is_personal = db.Column(db.Boolean, default=False)
"""Personal accesss token."""
is_internal = db.Column(db.Boolean, default=False)
"""Determines if token is an internally generated token."""
@property
def scopes(self):
"""Return all scopes."""
if self._scopes:
return self._scopes.split()
return []
@scopes.setter
def scopes(self, scopes):
"""Set scopes."""
validate_scopes(scopes)
self._scopes = " ".join(set(scopes)) if scopes else ""
def get_visible_scopes(self):
"""Get list of non-internal scopes for token."""
from .registry import scopes as scopes_registry
return [k for k, s in scopes_registry.choices() if k in self.scopes]
@classmethod
def create_personal(cls, name, user_id, scopes=None, is_internal=False):
"""Create a personal access token.
A token that is bound to a specific user and which doesn't expire, i.e.
similar to the concept of an API key.
"""
scopes = " ".join(scopes) if scopes else ""
c = Client(
name=name,
user_id=user_id,
is_internal=True,
is_confidential=False,
_default_scopes=scopes
)
c.gen_salt()
t = Token(
client_id=c.client_id,
user_id=user_id,
access_token=gen_salt(
current_app.config.get('OAUTH2_TOKEN_PERSONAL_SALT_LEN')
),
expires=None,
_scopes=scopes,
is_personal=True,
is_internal=is_internal,
)
db.session.add(c)
db.session.add(t)
db.session.commit()
return t
class RemoteAccount(db.Model):
"""Storage for remote linked accounts."""
__tablename__ = 'remoteACCOUNT'
__table_args__ = (db.UniqueConstraint('user_id', 'client_id'),
db.Model.__table_args__)
#
# Fields
#
id = db.Column(db.Integer(15, unsigned=True),
primary_key=True,
autoincrement=True)
"""Primary key."""
user_id = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=False)
"""Local user linked with a remote app via the access token."""
client_id = db.Column(db.String(255), nullable=False)
"""Client ID of remote application (defined in OAUTHCLIENT_REMOTE_APPS)."""
extra_data = db.Column(MutableDict.as_mutable(db.JSON), nullable=False)
"""Extra data associated with this linked account."""
#
# Relationships propoerties
#
user = db.relationship('User')
"""SQLAlchemy relationship to user."""
tokens = db.relationship(
"RemoteToken",
backref="remote_account",
)
"""SQLAlchemy relationship to RemoteToken objects."""
@classmethod
def get(cls, user_id, client_id):
"""Get RemoteAccount object for user.
:param user_id: User id
:param client_id: Client id.
"""
return cls.query.filter_by(
user_id=user_id,
client_id=client_id,
).first()
@classmethod
def create(cls, user_id, client_id, extra_data):
"""Create new remote account for user.
:param user_id: User id.
:param client_id: Client id.
:param extra_data: JSON-serializable dictionary of any extra data that
needs to be save together with this link.
"""
account = cls(user_id=user_id,
client_id=client_id,
extra_data=extra_data or dict())
db.session.add(account)
db.session.commit()
return account
def delete(self):
"""Delete remote account together with all stored tokens."""
RemoteToken.query.filter_by(id_remote_account=self.id).delete()
db.session.delete(self)
db.session.commit()
class RemoteToken(db.Model):
"""Storage for the access tokens for linked accounts."""
__tablename__ = 'remoteTOKEN'
#
# Fields
#
id_remote_account = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(RemoteAccount.id),
nullable=False,
primary_key=True)
"""Foreign key to account."""
token_type = db.Column(db.String(40),
default='',
nullable=False,
primary_key=True)
"""Type of token."""
access_token = db.Column(TextEncryptedType(type_in=db.Text,
key=secret_key),
nullable=False)
"""Access token to remote application."""
secret = db.Column(db.Text(), default='', nullable=False)
"""Used only by OAuth 1."""
def token(self):
"""Get token as expected by Flask-OAuthlib."""
return (self.access_token, self.secret)
def update_token(self, token, secret):
"""Update token with new values."""
if self.access_token != token or self.secret != secret:
self.access_token = token
self.secret = secret
db.session.commit()
@classmethod
def get(cls, user_id, client_id, token_type='', access_token=None):
"""Get RemoteToken for user."""
args = [
RemoteAccount.id == RemoteToken.id_remote_account,
RemoteAccount.user_id == user_id,
RemoteAccount.client_id == client_id,
RemoteToken.token_type == token_type,
]
if access_token:
args.append(RemoteToken.access_token == access_token)
return cls.query.options(
db.joinedload('remote_account')).filter(*args).first()
@classmethod
def get_by_token(cls, client_id, access_token, token_type=''):
"""Get RemoteAccount object for token."""
return cls.query.options(db.joinedload('remote_account')).filter(
RemoteAccount.id == RemoteToken.id_remote_account,
RemoteAccount.client_id == client_id,
RemoteToken.token_type == token_type,
RemoteToken.access_token == access_token,
).first()
@classmethod
def create(cls,
user_id,
client_id,
token,
secret,
token_type='',
extra_data=None):
"""Create a new access token.
Creates RemoteAccount as well if it does not exists.
"""
account = RemoteAccount.get(user_id, client_id)
if account is None:
account = RemoteAccount(
user_id=user_id,
client_id=client_id,
extra_data=extra_data or dict(),
)
db.session.add(account)
token = cls(
token_type=token_type,
remote_account=account,
access_token=token,
secret=secret,
)
db.session.add(token)
db.session.commit()
return token
class CheckerRule(db.Model):
"""Represent runnable rules (also known as tasks)."""
__tablename__ = 'checker_rule'
name = db.Column(
db.String(127),
primary_key=True,
doc="Name of the rule. Must be unique and user-friendly.",
)
plugin = db.Column(
db.String(127),
nullable=False,
doc="Check to use. Must be importable string. Does not need to exist"
" at task insertion time.",
)
arguments = db.Column(
JsonEncodedDict(1023),
default={},
doc="Arguments to pass to the check.",
)
# XXX: Currently unsupported by search. Disabled elsewhere in the code.
consider_deleted_records = db.Column(
db.Boolean,
nullable=True,
default=False,
doc="Whether to consider deleted records while filtering.",
)
filter_pattern = db.Column(
db.String(255),
nullable=True,
doc="String pattern to search with to resolve records to check.",
)
filter_records = db.Column(
IntBitSetType(1023),
nullable=True,
doc="Record IDs to run this task on.",
)
records = db.relationship(
'CheckerRecord',
backref='rule',
cascade='all, delete-orphan',
doc="Records which this rule has worked with in the past.",
)
reporters = db.relationship(
'CheckerReporter',
backref='rule',
cascade='all, delete-orphan',
doc="Reporters to be called while this task executes.",
)
executions = db.relationship(
'CheckerRuleExecution',
backref='rule',
cascade='all, delete-orphan',
doc="Past executions of this task. User should be free to clear them.",
)
last_scheduled_run = db.Column(
db.DateTime(),
nullable=True,
doc="Last time this task was ran by the scheduler.",
)
schedule = db.Column(
db.String(255),
nullable=True,
doc="Cron-style string that defines the schedule for this task.",
)
schedule_enabled = db.Column(
db.Boolean,
default=True,
nullable=False,
doc="Whether `schedule` is enabled.",
)
# TODO: You may use this column as a filter for tasks you don't want to see
# by default in interfaces
temporary = db.Column(
db.Boolean,
default=False,
doc="Flag for tasks which will not be reused.",
)
force_run_on_unmodified_records = db.Column(
db.Boolean,
default=False,
doc="Force a record-centric task to run on records it has checked"
" before, even if they have already been checked in their current"
" version.",
)
confirm_hash_on_commit = db.Column(
db.Boolean,
default=False,
doc="Only commit recids whose hash has not changed between first"
" requested modification and commit time.",
)
allow_chunking = db.Column( # XXX unclear name (maybe "run_in_parallel")
db.Boolean,
default=True,
doc="If the check is record-centric, allow checks to run in parallel.",
)
last_modification_date = db.Column(
db.DateTime(),
nullable=False,
server_default='1900-01-01 00:00:00',
doc="Last date on which this task was modified.",
)
owner_id = db.Column(
db.Integer(15, unsigned=True),
db.ForeignKey('user.id'),
nullable=False,
default=1,
)
owner = db.relationship(
'User',
doc="User that created this task. Used for scheduled tasks.",
)
@db.hybrid_property
def filepath(self):
"""Resolve a the filepath of this rule's plugin/check file."""
try:
path = inspect.getfile(plugin_files[self.plugin])
except KeyError:
return None
if path.endswith('.pyc'):
path = path[:-1]
return path
@db.hybrid_property
def modified_requested_recids(self):
"""Record IDs of records that match the filters of this task.
This property takes (0) `requested_ids`, (1) `filter_pattern` and if
`force_run_on_unmodified_records` is enabled (2)
`CheckerRecord.last_run_version_id` into consideration to figure out
which recids a record-centric task should run on.
:rtype: intbitset
"""
# Get all records that are already associated to this rule
# If this is returning an empty set, you forgot to run bibindex
try:
associated_records = intbitset(
zip(*db.session.query(CheckerRecord.rec_id).filter(
CheckerRecord.rule_name == self.name).all())[0])
except IndexError:
associated_records = intbitset()
# Store requested records that were until now unknown to this rule
requested_ids = self.requested_recids
for requested_id in requested_ids - associated_records:
new_record = CheckerRecord(rec_id=requested_id,
rule_name=self.name)
db.session.add(new_record)
db.session.commit()
# Figure out which records have been edited since the last time we ran
# this rule
try:
recids = zip(*db.session.query(CheckerRecord.rec_id).outerjoin(
RecordMetadata).filter(
CheckerRecord.rec_id.in_(requested_ids),
CheckerRecord.rule_name == self.name,
db.or_(
self.force_run_on_unmodified_records,
db.or_(
CheckerRecord.last_run_version_id == 1,
CheckerRecord.last_run_version_id <
RecordMetadata.version_id,
),
)))[0]
except IndexError:
recids = set()
return intbitset(recids)
@session_manager
def mark_recids_as_checked(self, recids):
"""Mark the given recids as checked by this task at their current `version_id`."""
db.session.query(CheckerRecord).\
filter(
CheckerRecord.rec_id == RecordMetadata.id,
CheckerRecord.rule_name == self.name,
CheckerRecord.rec_id.in_(recids),
).\
update({"last_run_version_id": RecordMetadata.version_id},
synchronize_session=False)
@db.hybrid_property
def requested_recids(self):
"""Search given `self.filter_pattern` and `self.filter_records`.
:rtype: intbitset"""
# TODO: Use self.option_consider_deleted_records when it's available
pattern = self.filter_pattern or ''
recids = Query(pattern).search().recids
if self.filter_records is not None:
recids &= self.filter_records
return recids
def __str__(self):
name_len = len(self.name)
trails = 61 - name_len
return '\n'.join((
'=== Checker Task: {} {}'.format(self.name, trails * '='),
'* Name: {}'.format(self.name),
'* Plugin: {}'.format(self.plugin),
'* Arguments: {}'.format(self.arguments),
'* Consider deleted records: {}'.format(
self.consider_deleted_records),
'* Filter Pattern: {}'.format(self.filter_pattern),
'* Filter Records: {}'.format(ranges_str(self.filter_records)),
'* Last scheduled run: {}'.format(self.last_scheduled_run),
'* Schedule: {} [{}]'.format(
self.schedule,
'enabled' if self.schedule_enabled else 'disabled'),
'* Temporary: {}'.format(self.temporary),
'* Force-run on unmodified records: {}'.format(
self.force_run_on_unmodified_records),
'{}'.format(80 * '='),
))
@staticmethod
def update_time(mapper, connection, instance):
"""Update the `last_modification_date` to the current time."""
instance.last_modification_date = datetime.now()
