class Bibfmt(db.Model):
"""Represent a Bibfmt record."""
__tablename__ = 'bibfmt'
id_bibrec = db.Column(
db.MediumInteger(8, unsigned=True),
nullable=False,
server_default='0',
primary_key=True,
autoincrement=False)
format = db.Column(
db.String(10),
nullable=False,
server_default='',
primary_key=True,
index=True)
kind = db.Column(
db.String(10),
nullable=False,
server_default='',
index=True
)
last_updated = db.Column(
db.DateTime,
nullable=False,
server_default='1900-01-01 00:00:00',
index=True)
value = db.Column(db.iLargeBinary)
needs_2nd_pass = db.Column(db.TinyInteger(1), server_default='0')
def do_upgrade():
"""Upgrade recipe.
Adds two new columns (password_salt and password_scheme) and migrates
emails to password salt.
"""
op.add_column(
'user', db.Column('password_salt',
db.String(length=255),
nullable=True))
op.add_column(
'user',
db.Column('password_scheme', db.String(length=50), nullable=False))
# Temporary column needed for data migration
op.add_column('user', db.Column('new_password', db.String(length=255)))
# Migrate emails to password_salt
m = db.MetaData(bind=db.engine)
m.reflect()
u = m.tables['user']
conn = db.engine.connect()
conn.execute(
u.update().values(password_salt=u.c.email,
password_scheme='invenio_aes_encrypted_email'))
# Migrate password blob to password varchar.
for row in conn.execute(select([u])):
# NOTE: Empty string passwords were stored as empty strings
# instead of a hashed version, hence they must be treated differently.
legacy_pw = row[u.c.password] or mysql_aes_encrypt(row[u.c.email], "")
stmt = u.update().where(u.c.id == row[u.c.id]).values(
new_password=hashlib.sha256(legacy_pw).hexdigest())
conn.execute(stmt)
# Create index
op.create_index(op.f('ix_user_password_scheme'),
'user', ['password_scheme'],
unique=False)
# Drop old database column and rename new.
op.drop_column('user', 'password')
op.alter_column(
'user',
'new_password',
new_column_name='password',
existing_type=mysql.VARCHAR(255),
existing_nullable=True,
)
class AccARGUMENT(db.Model):
"""Represent an authorization argument."""
__tablename__ = 'accARGUMENT'
id = db.Column(db.Integer(15), primary_key=True, autoincrement=True)
keyword = db.Column(db.String(32), nullable=True)
value = db.Column(db.String(255), nullable=True)
__table_args__ = (db.Index('KEYVAL', keyword, value),
db.Model.__table_args__)
def __repr__(self):
"""Repr."""
return "{0.keyword}={0.value}".format(self)
class AccROLE(db.Model):
"""Represent an access role."""
__tablename__ = 'accROLE'
id = db.Column(db.Integer(15, unsigned=True), primary_key=True,
autoincrement=True)
name = db.Column(db.String(32), unique=True, nullable=True)
description = db.Column(db.String(255), nullable=True)
firerole_def_ser = db.Column(db.iBinary, nullable=True)
firerole_def_src = db.Column(db.Text, nullable=True)
def __repr__(self):
"""Repr."""
return "{0.name} - {0.description}".format(self)
class Session(db.Model):
"""Represent Session record."""
__tablename__ = 'session'
session_key = db.Column(db.String(32),
nullable=False,
server_default='',
primary_key=True)
session_expiry = db.Column(db.DateTime, nullable=True, index=True)
session_object = db.Column(db.LargeBinary, nullable=True)
uid = db.Column(db.Integer(15, unsigned=True), nullable=False, index=True)
def get_session(self, name, expired=False):
"""Return an instance of :class:`Session`."""
where = Session.session_key == name
if expired:
where = db.and_(
where, Session.session_expiry >= db.func.current_timestamp())
return self.query.filter(where).one()
def set_session(self, name, value, timeout=None):
"""Store value in database."""
uid = current_user.get_id()
session_expiry = datetime.utcnow() + timeout
return Session(session_key=name,
session_object=value,
session_expiry=session_expiry,
uid=uid)
class UserQuery(db.Model):
"""Represent a UserQuery record."""
__tablename__ = 'user_query'
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
primary_key=True,
server_default='0')
id_query = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(WebQuery.id),
primary_key=True,
index=True,
server_default='0')
hostname = db.Column(db.String(50),
nullable=True,
server_default='unknown host')
date = db.Column(db.DateTime, nullable=True, default=datetime.datetime.now)
webquery = db.relationship(WebQuery, backref='executions')
@classmethod
def log(cls, urlargs=None, id_user=None):
"""Log."""
id_user = id_user if not None else current_user.get_id()
urlargs = urlargs or request.query_string
if id_user < 0:
return
webquery = WebQuery.query.filter_by(urlargs=urlargs).first()
if webquery is None:
webquery = WebQuery(urlargs=urlargs)
db.session.add(
cls(id_user=id_user, hostname=request.host, webquery=webquery))
db.session.commit()
class CheckerRecord(db.Model):
"""Connect checks with their executions on records."""
__tablename__ = 'checker_record'
rec_id = db.Column(
db.MediumInteger(8, unsigned=True),
db.ForeignKey(RecordMetadata.id),
primary_key=True,
nullable=False,
autoincrement=True,
)
record = db.relationship(
RecordMetadata,
backref=backref("checker_record", cascade="all, delete-orphan"),
doc="The record associated with a task.",
)
rule_name = db.Column(db.String(127),
db.ForeignKey('checker_rule.name'),
nullable=False,
index=True,
primary_key=True,
doc="Name of the task in this associaton.")
last_run_version_id = db.Column(
db.Integer,
nullable=False,
doc="Last checked version ID of associated record.",
)
class PidLog(db.Model):
"""Audit log of actions happening to persistent identifiers.
This model is primarily used through PersistentIdentifier.log and rarely
created manually.
"""
__tablename__ = 'pidLOG'
__table_args__ = (
db.Index('idx_action', 'action'),
)
id = db.Column(db.Integer(15, unsigned=True), primary_key=True)
"""Id of persistent identifier entry."""
id_pid = db.Column(
db.Integer(15, unsigned=True), db.ForeignKey(PersistentIdentifier.id),
nullable=True,
)
"""PID."""
timestamp = db.Column(db.DateTime(), nullable=False, default=datetime.now)
"""Creation datetime of entry."""
action = db.Column(db.String(10), nullable=False)
"""Action identifier."""
message = db.Column(db.Text(), nullable=False)
"""Log message."""
# Relationship
pid = db.relationship("PersistentIdentifier", backref="logs")
class AccACTION(db.Model):
"""Represent an access action."""
__tablename__ = 'accACTION'
id = db.Column(db.Integer(15, unsigned=True),
primary_key=True, autoincrement=True)
name = db.Column(db.String(32), unique=True, nullable=True)
description = db.Column(db.String(255), nullable=True)
allowedkeywords = db.Column(db.String(255), nullable=True)
optional = db.Column(db.Enum('yes', 'no', name='yes_no'), nullable=False,
server_default='no')
def __repr__(self):
"""Repr."""
return "{0.name}".format(self)
class UserEXT(db.Model):
"""Represent a UserEXT record."""
__tablename__ = 'userEXT'
id = db.Column(db.String(255), primary_key=True, nullable=False)
method = db.Column(db.String(50), primary_key=True, nullable=False)
id_user = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=False)
user = db.relationship(User, backref="external_identifiers")
__table_args__ = (db.Index('userext_id_user_method',
id_user,
method,
unique=True), db.Model.__table_args__)
def do_upgrade():
""" Implement your upgrades here """
if not op.has_table('remoteACCOUNT'):
op.create_table('remoteACCOUNT',
db.Column('id',
db.Integer(display_width=15),
nullable=False),
db.Column('user_id',
db.Integer(display_width=15),
nullable=False),
db.Column('client_id',
db.String(length=255),
nullable=False),
db.Column('extra_data', db.JSON, nullable=True),
db.ForeignKeyConstraint(
['user_id'],
['user.id'],
),
db.PrimaryKeyConstraint('id'),
db.UniqueConstraint('user_id', 'client_id'),
mysql_charset='utf8',
mysql_engine='MyISAM')
else:
warnings.warn("*** Creation of table 'remoteACCOUNT table skipped!'")
if not op.has_table('remoteTOKEN'):
op.create_table('remoteTOKEN',
db.Column('id_remote_account',
db.Integer(display_width=15),
nullable=False),
db.Column('token_type',
db.String(length=40),
nullable=False),
db.Column('access_token', db.Text(), nullable=False),
db.Column('secret', db.Text(), nullable=False),
db.ForeignKeyConstraint(
['id_remote_account'],
['remoteACCOUNT.id'],
),
db.PrimaryKeyConstraint('id_remote_account',
'token_type'),
mysql_charset='utf8',
mysql_engine='MyISAM')
else:
warnings.warn("*** Creation of table 'remoteTOKEN' skipped!'")
def do_upgrade():
"""Carry out the upgrade."""
op.alter_column(table_name='oauth2TOKEN',
column_name='client_id',
type_=db.String(255),
existing_nullable=False)
op.alter_column(table_name='oauth2TOKEN',
column_name='user_id',
type_=db.Integer(15, unsigned=True),
existing_nullable=False)
class Fieldname(db.Model):
"""Represent a Fieldname record."""
__tablename__ = 'fieldname'
id_field = db.Column(db.MediumInteger(9, unsigned=True),
db.ForeignKey(Field.id),
primary_key=True)
ln = db.Column(db.Char(5), primary_key=True, server_default='')
type = db.Column(db.Char(3), primary_key=True, server_default='sn')
value = db.Column(db.String(255), nullable=False)
field = db.relationship(Field, backref='names')
class Fieldvalue(db.Model):
"""Represent a Fieldvalue record."""
def __init__(self):
"""Init."""
pass
__tablename__ = 'fieldvalue'
id = db.Column(db.MediumInteger(9, unsigned=True),
primary_key=True,
autoincrement=True)
name = db.Column(db.String(255), nullable=False)
value = db.Column(db.Text, nullable=False)
class Goto(db.Model):
"""Represents a Goto record."""
__tablename__ = 'goto'
label = db.Column(db.String(150), primary_key=True)
plugin = db.Column(db.String(150), nullable=False)
_parameters = db.Column(db.JSON, nullable=False, default={},
name="parameters")
creation_date = db.Column(db.DateTime, default=datetime.datetime.now,
nullable=False, index=True)
modification_date = db.Column(db.DateTime, default=datetime.datetime.now,
onupdate=datetime.datetime.now,
nullable=False, index=True)
@validates('plugin')
def validate_plugin(self, key, plugin):
"""Validate plugin name."""
if plugin not in redirect_methods:
raise ValueError("%s plugin does not exist" % plugin)
return plugin
@db.hybrid_property
def parameters(self):
"""Get parameters method."""
return self._parameters
@parameters.setter
def parameters(self, value):
"""Set parameters method."""
self._parameters = value or {}
def to_dict(self):
"""Return a dict representation of Goto."""
return {'label': self.label,
'plugin': self.plugin,
'parameters': self.parameters,
'creation_date': self.creation_date,
'modification_date': self.modification_date}
class Field(db.Model):
"""Represent a Field record."""
def __repr__(self):
"""Get repr."""
return "%s(%s)" % (self.__class__.__name__, self.id)
__tablename__ = 'field'
id = db.Column(db.MediumInteger(9, unsigned=True), primary_key=True)
name = db.Column(db.String(255), nullable=False)
code = db.Column(db.String(255), unique=True, nullable=False)
@property
def name_ln(self):
"""Get name ln."""
from .cache import get_field_i18nname
return get_field_i18nname(self.name,
getattr(g, 'ln', cfg['CFG_SITE_LANG']))
# try:
# return db.object_session(self).query(Fieldname).\
# with_parent(self).filter(db.and_(Fieldname.ln==g.ln,
# Fieldname.type=='ln')).first().value
# except Exception:
# return self.name
@classmethod
def get_field_name(cls, code):
"""Return field name for given code."""
return cls.query.filter_by(code=code).value(cls.name)
@classmethod
def get_field_tags(cls, code, tagtype='marc'):
"""Yield tag values for given field code."""
column = Tag.value if tagtype == 'marc' else Tag.recjson_value
tags = cls.query.join(cls.tags).join(
FieldTag.tag).filter(cls.code == code).values(column)
for tag in tags:
for value in tag[0].split(','):
yield value.strip()
class CheckerReporter(db.Model):
"""Represent reporters associated with a task.
..note::
These entries are currently not meant to be associated with multiple
tasks, because it is assumed that they may be deleted without affecting
more than one tasks.
"""
__tablename__ = 'checker_reporter'
plugin = db.Column(db.String(127),
primary_key=True,
doc="Check associated with this reporter."
"")
rule_name = db.Column(db.String(127),
db.ForeignKey('checker_rule.name',
onupdate="CASCADE",
ondelete="CASCADE"),
index=True,
nullable=False,
primary_key=True,
doc="Task associated with this reporter."
"")
arguments = db.Column(
JsonEncodedDict(1023),
default={},
doc="Arguments to be passed to this reporter.",
)
@db.hybrid_property
def module(self):
"""Python module of the associated check."""
return reporters_files[self.plugin]
def do_upgrade():
"""Add the table with facets configuration."""
if not op.has_table('facet_collection'):
op.create_table(
'facet_collection',
db.Column('id', mysql.INTEGER(), nullable=False),
db.Column('id_collection', mysql.INTEGER(), nullable=False),
db.Column('order', mysql.INTEGER(), nullable=False),
db.Column('facet_name', db.String(length=80), nullable=False),
db.ForeignKeyConstraint(['id_collection'], ['collection.id'], ),
db.PrimaryKeyConstraint('id'),
mysql_charset='utf8',
mysql_engine='MyISAM'
)
else:
warnings.warn("*** Creation of table 'facet_collection' skipped!")
class Tag(db.Model):
"""Represent a Tag record."""
__tablename__ = 'tag'
id = db.Column(db.MediumInteger(9, unsigned=True), primary_key=True)
name = db.Column(db.String(255), nullable=False)
value = db.Column(db.Char(6), nullable=False, server_default='')
recjson_value = db.Column(db.Text, nullable=False)
def __init__(self, tup=None, *args, **kwargs):
"""Init."""
if tup is not None and isinstance(tup, tuple):
self.name, self.value = tup
super(Tag, self).__init__(*args, **kwargs)
else:
if tup is None:
super(Tag, self).__init__(*args, **kwargs)
else:
super(Tag, self).__init__(tup, *args, **kwargs)
@property
def as_tag(self):
"""Return tupple with name and value."""
return self.name, self.value
class KnwKBRVAL(db.Model):
"""Represent a KnwKBRVAL record."""
__tablename__ = 'knwKBRVAL'
m_key = db.Column(db.String(255),
nullable=False,
primary_key=True,
index=True)
m_value = db.Column(db.Text().with_variant(mysql.TEXT(30), 'mysql'),
nullable=False)
id_knwKB = db.Column(db.MediumInteger(8, unsigned=True),
db.ForeignKey(KnwKB.id),
nullable=False,
server_default='0',
primary_key=True)
kb = db.relationship(
KnwKB,
backref=db.backref(
'kbrvals',
cascade="all, delete-orphan",
collection_class=attribute_mapped_collection("m_key")))
@staticmethod
def query_kb_mappings(kbid, sortby="to", key="", value="", match_type="s"):
"""Return a list of all mappings from the given kb, ordered by key.
If key given, give only those with left side (mapFrom) = key.
If value given, give only those with right side (mapTo) = value.
:param kb_name: knowledge base name. if "", return all
:param sortby: the sorting criteria ('from' or 'to')
:param key: return only entries where key matches this
:param value: return only entries where value matches this
:param match_type: s=substring, e=exact, sw=startswith
"""
# query
query = KnwKBRVAL.query.filter(KnwKBRVAL.id_knwKB == kbid)
# filter
if len(key) > 0:
if match_type == "s":
key = "%" + key + "%"
elif match_type == "sw":
key = key + "%"
else:
key = '%'
if len(value) > 0:
if match_type == "s":
value = "%" + value + "%"
elif match_type == "sw":
value = value + "%"
else:
value = '%'
query = query.filter(KnwKBRVAL.m_key.like(key),
KnwKBRVAL.m_value.like(value))
# order by
if sortby == "from":
query = query.order_by(KnwKBRVAL.m_key)
else:
query = query.order_by(KnwKBRVAL.m_value)
return query
def to_dict(self):
"""Return a dict representation of KnwKBRVAL."""
# FIXME remove 'id' dependency from invenio modules
return {
'id': self.m_key + "_" + str(self.id_knwKB),
'key': self.m_key,
'value': self.m_value,
'kbid': self.kb.id if self.kb else None,
'kbname': self.kb.name if self.kb else None
}
class CheckerRule(db.Model):
"""Represent runnable rules (also known as tasks)."""
__tablename__ = 'checker_rule'
name = db.Column(
db.String(127),
primary_key=True,
doc="Name of the rule. Must be unique and user-friendly.",
)
plugin = db.Column(
db.String(127),
nullable=False,
doc="Check to use. Must be importable string. Does not need to exist"
" at task insertion time.",
)
arguments = db.Column(
JsonEncodedDict(1023),
default={},
doc="Arguments to pass to the check.",
)
# XXX: Currently unsupported by search. Disabled elsewhere in the code.
consider_deleted_records = db.Column(
db.Boolean,
nullable=True,
default=False,
doc="Whether to consider deleted records while filtering.",
)
filter_pattern = db.Column(
db.String(255),
nullable=True,
doc="String pattern to search with to resolve records to check.",
)
filter_records = db.Column(
IntBitSetType(1023),
nullable=True,
doc="Record IDs to run this task on.",
)
records = db.relationship(
'CheckerRecord',
backref='rule',
cascade='all, delete-orphan',
doc="Records which this rule has worked with in the past.",
)
reporters = db.relationship(
'CheckerReporter',
backref='rule',
cascade='all, delete-orphan',
doc="Reporters to be called while this task executes.",
)
executions = db.relationship(
'CheckerRuleExecution',
backref='rule',
cascade='all, delete-orphan',
doc="Past executions of this task. User should be free to clear them.",
)
last_scheduled_run = db.Column(
db.DateTime(),
nullable=True,
doc="Last time this task was ran by the scheduler.",
)
schedule = db.Column(
db.String(255),
nullable=True,
doc="Cron-style string that defines the schedule for this task.",
)
schedule_enabled = db.Column(
db.Boolean,
default=True,
nullable=False,
doc="Whether `schedule` is enabled.",
)
# TODO: You may use this column as a filter for tasks you don't want to see
# by default in interfaces
temporary = db.Column(
db.Boolean,
default=False,
doc="Flag for tasks which will not be reused.",
)
force_run_on_unmodified_records = db.Column(
db.Boolean,
default=False,
doc="Force a record-centric task to run on records it has checked"
" before, even if they have already been checked in their current"
" version.",
)
confirm_hash_on_commit = db.Column(
db.Boolean,
default=False,
doc="Only commit recids whose hash has not changed between first"
" requested modification and commit time.",
)
allow_chunking = db.Column( # XXX unclear name (maybe "run_in_parallel")
db.Boolean,
default=True,
doc="If the check is record-centric, allow checks to run in parallel.",
)
last_modification_date = db.Column(
db.DateTime(),
nullable=False,
server_default='1900-01-01 00:00:00',
doc="Last date on which this task was modified.",
)
owner_id = db.Column(
db.Integer(15, unsigned=True),
db.ForeignKey('user.id'),
nullable=False,
default=1,
)
owner = db.relationship(
'User',
doc="User that created this task. Used for scheduled tasks.",
)
@db.hybrid_property
def filepath(self):
"""Resolve a the filepath of this rule's plugin/check file."""
try:
path = inspect.getfile(plugin_files[self.plugin])
except KeyError:
return None
if path.endswith('.pyc'):
path = path[:-1]
return path
@db.hybrid_property
def modified_requested_recids(self):
"""Record IDs of records that match the filters of this task.
This property takes (0) `requested_ids`, (1) `filter_pattern` and if
`force_run_on_unmodified_records` is enabled (2)
`CheckerRecord.last_run_version_id` into consideration to figure out
which recids a record-centric task should run on.
:rtype: intbitset
"""
# Get all records that are already associated to this rule
# If this is returning an empty set, you forgot to run bibindex
try:
associated_records = intbitset(
zip(*db.session.query(CheckerRecord.rec_id).filter(
CheckerRecord.rule_name == self.name).all())[0])
except IndexError:
associated_records = intbitset()
# Store requested records that were until now unknown to this rule
requested_ids = self.requested_recids
for requested_id in requested_ids - associated_records:
new_record = CheckerRecord(rec_id=requested_id,
rule_name=self.name)
db.session.add(new_record)
db.session.commit()
# Figure out which records have been edited since the last time we ran
# this rule
try:
recids = zip(*db.session.query(CheckerRecord.rec_id).outerjoin(
RecordMetadata).filter(
CheckerRecord.rec_id.in_(requested_ids),
CheckerRecord.rule_name == self.name,
db.or_(
self.force_run_on_unmodified_records,
db.or_(
CheckerRecord.last_run_version_id == 1,
CheckerRecord.last_run_version_id <
RecordMetadata.version_id,
),
)))[0]
except IndexError:
recids = set()
return intbitset(recids)
@session_manager
def mark_recids_as_checked(self, recids):
"""Mark the given recids as checked by this task at their current `version_id`."""
db.session.query(CheckerRecord).\
filter(
CheckerRecord.rec_id == RecordMetadata.id,
CheckerRecord.rule_name == self.name,
CheckerRecord.rec_id.in_(recids),
).\
update({"last_run_version_id": RecordMetadata.version_id},
synchronize_session=False)
@db.hybrid_property
def requested_recids(self):
"""Search given `self.filter_pattern` and `self.filter_records`.
:rtype: intbitset"""
# TODO: Use self.option_consider_deleted_records when it's available
pattern = self.filter_pattern or ''
recids = Query(pattern).search().recids
if self.filter_records is not None:
recids &= self.filter_records
return recids
def __str__(self):
name_len = len(self.name)
trails = 61 - name_len
return '\n'.join((
'=== Checker Task: {} {}'.format(self.name, trails * '='),
'* Name: {}'.format(self.name),
'* Plugin: {}'.format(self.plugin),
'* Arguments: {}'.format(self.arguments),
'* Consider deleted records: {}'.format(
self.consider_deleted_records),
'* Filter Pattern: {}'.format(self.filter_pattern),
'* Filter Records: {}'.format(ranges_str(self.filter_records)),
'* Last scheduled run: {}'.format(self.last_scheduled_run),
'* Schedule: {} [{}]'.format(
self.schedule,
'enabled' if self.schedule_enabled else 'disabled'),
'* Temporary: {}'.format(self.temporary),
'* Force-run on unmodified records: {}'.format(
self.force_run_on_unmodified_records),
'{}'.format(80 * '='),
))
@staticmethod
def update_time(mapper, connection, instance):
"""Update the `last_modification_date` to the current time."""
instance.last_modification_date = datetime.now()
class Record(db.Model):
"""Represent a record object inside the SQL database."""
__tablename__ = 'bibrec'
id = db.Column(db.MediumInteger(8, unsigned=True),
primary_key=True,
nullable=False,
autoincrement=True)
creation_date = db.Column(db.DateTime,
nullable=False,
server_default='1900-01-01 00:00:00',
index=True)
modification_date = db.Column(db.DateTime,
nullable=False,
server_default='1900-01-01 00:00:00',
index=True)
master_format = db.Column(db.String(16),
nullable=False,
server_default='marc')
additional_info = db.Column(db.JSON)
# FIXME: remove this from the model and add them to the record class, all?
@property
def deleted(self):
"""Return True if record is marked as deleted."""
from .api import get_record
dbcollids = [
c.get('primary') for c in get_record(self.id).get('collections')
]
# record exists; now check whether it isn't marked as deleted:
return ("DELETED" in dbcollids) or \
(current_app.config.get('CFG_CERN_SITE') and
"DUMMY" in dbcollids)
@staticmethod
def _next_merged_recid(recid):
"""Return the ID of record merged with record with ID = recid."""
from .api import get_record
merged_recid = None
# FIXME
for val in get_record(recid).get("970__d", []):
try:
merged_recid = int(val)
break
except ValueError:
pass
if not merged_recid:
return None
else:
return merged_recid
@cached_property
def merged_recid(self):
"""Return record object with which the given record has been merged.
:param recID: deleted record recID
:return: merged record recID
"""
return Record._next_merged_recid(self.id)
@property
def merged_recid_final(self):
"""Return the last record from hierarchy merged with this one."""
cur_id = self.id
next_id = Record._next_merged_recid(cur_id)
while next_id:
cur_id = next_id
next_id = Record._next_merged_recid(cur_id)
return cur_id
@classmethod
def filter_time_interval(cls, datetext, column='c'):
"""Return filter based on date text and column type."""
column = cls.creation_date if column == 'c' else cls.modification_date
parts = datetext.split('->')
where = []
if len(parts) == 2:
if parts[0] != '':
where.append(column >= parts[0])
if parts[1] != '':
where.append(column <= parts[1])
else:
where.append(column.like(datetext + '%'))
return where
@classmethod
def allids(cls):
"""Return all existing record ids."""
return intbitset(db.session.query(cls.id).all())
def do_upgrade():
"""Implement your upgrades here."""
op.add_column(
'user', db.Column('family_name', db.String(length=255), nullable=True))
op.add_column(
'user', db.Column('given_names', db.String(length=255), nullable=True))
class User(db.Model):
"""Represents a User record."""
def __str__(self):
"""Return string representation."""
return "%s <%s>" % (self.nickname, self.email)
__tablename__ = 'user'
__mapper_args__ = {'confirm_deleted_rows': False}
id = db.Column(db.Integer(15, unsigned=True),
primary_key=True,
autoincrement=True)
email = db.Column(db.String(255),
nullable=False,
server_default='',
index=True)
_password = db.Column(db.String(255), name="password", nullable=True)
password_salt = db.Column(db.String(255))
password_scheme = db.Column(db.String(50), nullable=False, index=True)
_note = db.Column(db.String(255), name="note", nullable=True)
given_names = db.Column(db.String(255), nullable=False, server_default='')
family_name = db.Column(db.String(255), nullable=False, server_default='')
settings = db.Column(db.MutableDict.as_mutable(
db.MarshalBinary(default_value=get_default_user_preferences,
force_type=dict)),
nullable=True)
nickname = db.Column(db.String(255),
nullable=False,
server_default='',
index=True)
last_login = db.Column(db.DateTime,
nullable=False,
server_default='1900-01-01 00:00:00')
PROFILE_FIELDS = ['nickname', 'email', 'family_name', 'given_names']
"""List of fields that can be updated with update_profile."""
@staticmethod
def check_nickname(nickname):
"""Check if it's a valid nickname."""
re_invalid_nickname = re.compile(""".*[,'@]+.*""")
return bool(nickname) and not nickname.startswith(' ') and \
not nickname.endswith(' ') and \
nickname.lower() != 'guest' and \
not re_invalid_nickname.match(nickname)
@staticmethod
def check_email(email):
"""Check if it's a valid email."""
r = re.compile(r'(.)+\@(.)+\.(.)+')
return bool(email) and r.match(email) and not email.find(" ") > 0
@hybrid_property
def note(self):
"""Return the note."""
return self._note
@note.setter
def note(self, note):
"""Set the note."""
self._note = str(note)
@hybrid_property
def password(self):
"""Return the password."""
return self._password
@password.setter
def password(self, password):
"""Set the password."""
if password is None:
# Unusable password.
self._password = None
self.password_scheme = ''
else:
self._password = password_context.encrypt(password)
self.password_scheme = password_context.default_scheme()
# Invenio legacy salt is stored in password_salt, and every new
# password set will be migrated to new hash not relying on
# password_salt, thus is force to empty value.
self.password_salt = ""
def verify_password(self, password, migrate=False):
"""Verify if password matches the stored password hash."""
if self.password is None or password is None:
return False
# Invenio 1.x legacy needs externally store password salt to compute
# hash.
scheme_ctx = {} if \
self.password_scheme != invenio_aes_encrypted_email.name else \
{'user': self.password_salt}
# Verify password
if not password_context.verify(
password, self.password, scheme=self.password_scheme, **
scheme_ctx):
return False
# Migrate hash if needed.
if migrate and password_context.needs_update(self.password):
self.password = password
try:
db.session.commit()
except Exception:
db.session.rollback()
raise
return True
def verify_email(self, force=False):
"""Verify email address."""
if force or self.note == "2":
if self.note != "2":
self.note = 2
try:
db.session.commit()
except Exception:
db.session.rollback()
raise
send_account_activation_email(self)
return True
return False
def update_profile(self, data):
"""Update user profile.
Sends signal to allow other modules to subscribe to changes.
"""
changed_attrs = {}
for field in self.PROFILE_FIELDS:
if field in data and getattr(self, field) != data[field]:
changed_attrs[field] = getattr(self, field)
setattr(self, field, data[field])
if 'email' in changed_attrs:
self.verify_email(force=True)
try:
db.session.commit()
except Exception:
db.session.rollback()
raise
current_user.reload()
profile_updated.send(sender=self.id,
user=self,
changed_attrs=changed_attrs)
return changed_attrs
@property
def guest(self):
"""Return True if the user is a guest."""
return False if self.email else True
#
# Basic functions for user authentification.
#
def get_id(self):
"""Return the id."""
return self.id
def is_confirmed(self):
"""Return true if accounts has been confirmed."""
return self.note == "1"
def is_guest(self):
"""Return if the user is a guest."""
return self.guest
def is_authenticated(self):
"""Return True if user is a authenticated user."""
return True if self.email else False
def is_active(self):
"""Return True if use is active."""
return self.note != "0"
class AccMAILCOOKIE(db.Model):
"""Represent an email cookie."""
__tablename__ = 'accMAILCOOKIE'
AUTHORIZATIONS_KIND = (
'pw_reset', 'mail_activation', 'role', 'authorize_action',
'comment_msg', 'generic'
)
id = db.Column(db.Integer(15, unsigned=True), primary_key=True,
autoincrement=True)
_data = db.Column('data', db.iBinary, nullable=False)
expiration = db.Column(db.DateTime, nullable=False,
server_default='9999-12-31 23:59:59', index=True)
kind = db.Column(db.String(32), nullable=False)
onetime = db.Column(db.TinyInteger(1), nullable=False, server_default='0')
status = db.Column(db.Char(1), nullable=False, server_default='W')
@validates('kind')
def validate_kind(self, key, kind):
"""Validate cookie kind."""
assert kind in self.AUTHORIZATIONS_KIND
return kind
@classmethod
def get(cls, cookie, delete=False):
"""Get cookie if it is valid."""
password = cookie[:16]+cookie[-16:]
cookie_id = int(cookie[16:-16], 16)
obj, data = db.session.query(
cls,
AccMAILCOOKIE._data
).filter_by(id=cookie_id).one()
obj.data = loads(mysql_aes_decrypt(data, password))
(kind_check, params, expiration, onetime_check) = obj.data
assert obj.kind in cls.AUTHORIZATIONS_KIND
if not (obj.kind == kind_check and obj.onetime == onetime_check):
raise InvenioWebAccessMailCookieError("Cookie is corrupted")
if obj.status == 'D':
raise InvenioWebAccessMailCookieDeletedError(
"Cookie has been deleted")
if obj.onetime or delete:
obj.status = 'D'
db.session.merge(obj)
db.session.commit()
return obj
@classmethod
def create(cls, kind, params, cookie_timeout=timedelta(days=1),
onetime=False):
"""Create cookie with given params."""
expiration = datetime.today() + cookie_timeout
data = (kind, params, expiration, onetime)
password = md5(str(random())).hexdigest()
cookie = cls(
expiration=expiration,
kind=kind,
onetime=int(onetime),
)
cookie._data = mysql_aes_encrypt(dumps(data), password)
db.session.add(cookie)
db.session.commit()
db.session.refresh(cookie)
return password[:16]+hex(cookie.id)[2:-1]+password[-16:]
@classmethod
@session_manager
def gc(cls):
"""Remove expired items."""
return cls.query.filter(cls.expiration < db.func.now()).delete()
class RemoteToken(db.Model):
"""Storage for the access tokens for linked accounts."""
__tablename__ = 'remoteTOKEN'
#
# Fields
#
id_remote_account = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(RemoteAccount.id),
nullable=False,
primary_key=True)
"""Foreign key to account."""
token_type = db.Column(db.String(40),
default='',
nullable=False,
primary_key=True)
"""Type of token."""
access_token = db.Column(TextEncryptedType(type_in=db.Text,
key=secret_key),
nullable=False)
"""Access token to remote application."""
secret = db.Column(db.Text(), default='', nullable=False)
"""Used only by OAuth 1."""
def token(self):
"""Get token as expected by Flask-OAuthlib."""
return (self.access_token, self.secret)
def update_token(self, token, secret):
"""Update token with new values."""
if self.access_token != token or self.secret != secret:
self.access_token = token
self.secret = secret
db.session.commit()
@classmethod
def get(cls, user_id, client_id, token_type='', access_token=None):
"""Get RemoteToken for user."""
args = [
RemoteAccount.id == RemoteToken.id_remote_account,
RemoteAccount.user_id == user_id,
RemoteAccount.client_id == client_id,
RemoteToken.token_type == token_type,
]
if access_token:
args.append(RemoteToken.access_token == access_token)
return cls.query.options(
db.joinedload('remote_account')).filter(*args).first()
@classmethod
def get_by_token(cls, client_id, access_token, token_type=''):
"""Get RemoteAccount object for token."""
return cls.query.options(db.joinedload('remote_account')).filter(
RemoteAccount.id == RemoteToken.id_remote_account,
RemoteAccount.client_id == client_id,
RemoteToken.token_type == token_type,
RemoteToken.access_token == access_token,
).first()
@classmethod
def create(cls,
user_id,
client_id,
token,
secret,
token_type='',
extra_data=None):
"""Create a new access token.
Creates RemoteAccount as well if it does not exists.
"""
account = RemoteAccount.get(user_id, client_id)
if account is None:
account = RemoteAccount(
user_id=user_id,
client_id=client_id,
extra_data=extra_data or dict(),
)
db.session.add(account)
token = cls(
token_type=token_type,
remote_account=account,
access_token=token,
secret=secret,
)
db.session.add(token)
db.session.commit()
return token
class RemoteAccount(db.Model):
"""Storage for remote linked accounts."""
__tablename__ = 'remoteACCOUNT'
__table_args__ = (db.UniqueConstraint('user_id', 'client_id'),
db.Model.__table_args__)
#
# Fields
#
id = db.Column(db.Integer(15, unsigned=True),
primary_key=True,
autoincrement=True)
"""Primary key."""
user_id = db.Column(db.Integer(15, unsigned=True),
db.ForeignKey(User.id),
nullable=False)
"""Local user linked with a remote app via the access token."""
client_id = db.Column(db.String(255), nullable=False)
"""Client ID of remote application (defined in OAUTHCLIENT_REMOTE_APPS)."""
extra_data = db.Column(MutableDict.as_mutable(db.JSON), nullable=False)
"""Extra data associated with this linked account."""
#
# Relationships propoerties
#
user = db.relationship('User')
"""SQLAlchemy relationship to user."""
tokens = db.relationship(
"RemoteToken",
backref="remote_account",
)
"""SQLAlchemy relationship to RemoteToken objects."""
@classmethod
def get(cls, user_id, client_id):
"""Get RemoteAccount object for user.
:param user_id: User id
:param client_id: Client id.
"""
return cls.query.filter_by(
user_id=user_id,
client_id=client_id,
).first()
@classmethod
def create(cls, user_id, client_id, extra_data):
"""Create new remote account for user.
:param user_id: User id.
:param client_id: Client id.
:param extra_data: JSON-serializable dictionary of any extra data that
needs to be save together with this link.
"""
account = cls(user_id=user_id,
client_id=client_id,
extra_data=extra_data or dict())
db.session.add(account)
db.session.commit()
return account
def delete(self):
"""Delete remote account together with all stored tokens."""
RemoteToken.query.filter_by(id_remote_account=self.id).delete()
db.session.delete(self)
db.session.commit()
class KnwKB(db.Model):
"""Represent a KnwKB record."""
KNWKB_TYPES = {
'written_as': 'w',
'dynamic': 'd',
'taxonomy': 't',
}
__tablename__ = 'knwKB'
id = db.Column(db.MediumInteger(8, unsigned=True),
nullable=False,
primary_key=True,
autoincrement=True)
_name = db.Column(db.String(255),
server_default='',
unique=True,
name="name")
_description = db.Column(db.Text,
nullable=False,
name="description",
default="")
_kbtype = db.Column(db.Char(1), nullable=True, default='w', name="kbtype")
slug = db.Column(db.String(255), unique=True, nullable=False, default="")
# Enable or disable the access from REST API
is_api_accessible = db.Column(db.Boolean, default=True, nullable=False)
@db.hybrid_property
def name(self):
"""Get name."""
return self._name
@name.setter
def name(self, value):
"""Set name and generate the slug."""
self._name = value
# generate slug
if not self.slug:
self.slug = KnwKB.generate_slug(value)
@db.hybrid_property
def description(self):
"""Get description."""
return self._description
@description.setter
def description(self, value):
"""Set description."""
# TEXT in mysql don't support default value
# @see http://bugs.mysql.com/bug.php?id=21532
self._description = value or ''
@db.hybrid_property
def kbtype(self):
"""Get kbtype."""
return self._kbtype
@kbtype.setter
def kbtype(self, value):
"""Set kbtype."""
if value is None:
# set the default value
return
# or set one of the available values
kbtype = value[0] if len(value) > 0 else 'w'
if kbtype not in ['t', 'd', 'w']:
raise ValueError('unknown type "{value}", please use one of \
following values: "taxonomy", "dynamic" or \
"written_as"'.format(value=value))
self._kbtype = kbtype
def is_dynamic(self):
"""Return true if the type is dynamic."""
return self._kbtype == 'd'
def to_dict(self):
"""Return a dict representation of KnwKB."""
mydict = {
'id': self.id,
'name': self.name,
'description': self.description,
'kbtype': self.kbtype
}
if self.kbtype == 'd':
mydict.update((self.kbdefs.to_dict() if self.kbdefs else {}) or {})
return mydict
def get_kbr_items(self, searchkey="", searchvalue="", searchtype='s'):
"""
Return dicts of 'key' and 'value' from a knowledge base.
:param kb_name the name of the knowledge base
:param searchkey search using this key
:param searchvalue search using this value
:param searchtype s=substring, e=exact, sw=startswith
:return a list of dictionaries [{'key'=>x, 'value'=>y},..]
"""
import warnings
warnings.warn("The function is deprecated. Please use the "
"`KnwKBRVAL.query_kb_mappings()` instead. "
"E.g. [kval.to_dict() for kval in "
"KnwKBRVAL.query_kb_mappings(kb_id).all()]")
if searchtype == 's' and searchkey:
searchkey = '%' + searchkey + '%'
if searchtype == 's' and searchvalue:
searchvalue = '%' + searchvalue + '%'
if searchtype == 'sw' and searchvalue: # startswith
searchvalue = searchvalue + '%'
if not searchvalue:
searchvalue = '%'
if not searchkey:
searchkey = '%'
kvals = KnwKBRVAL.query.filter(KnwKBRVAL.id_knwKB.like(self.id),
KnwKBRVAL.m_value.like(searchvalue),
KnwKBRVAL.m_key.like(searchkey)).all()
return [kval.to_dict() for kval in kvals]
def get_kbr_values(self, searchkey="", searchvalue="", searchtype='s'):
"""
Return dicts of 'key' and 'value' from a knowledge base.
:param kb_name the name of the knowledge base
:param searchkey search using this key
:param searchvalue search using this value
:param searchtype s=substring, e=exact, sw=startswith
:return a list of dictionaries [{'key'=>x, 'value'=>y},..]
"""
import warnings
warnings.warn("The function is deprecated. Please use the "
"`KnwKBRVAL.query_kb_mappings()` instead. "
"E.g. [(kval.m_value,) for kval in "
"KnwKBRVAL.query_kb_mappings(kb_id).all()]")
# prepare filters
if searchtype == 's':
searchkey = '%' + searchkey + '%'
if searchtype == 's' and searchvalue:
searchvalue = '%' + searchvalue + '%'
if searchtype == 'sw' and searchvalue: # startswith
searchvalue = searchvalue + '%'
if not searchvalue:
searchvalue = '%'
# execute query
return db.session.execute(
db.select([KnwKBRVAL.m_value],
db.and_(KnwKBRVAL.id_knwKB.like(self.id),
KnwKBRVAL.m_value.like(searchvalue),
KnwKBRVAL.m_key.like(searchkey))))
@session_manager
def set_dyn_config(self, field, expression, collection=None):
"""Set dynamic configuration."""
if self.kbdefs:
# update
self.kbdefs.output_tag = field
self.kbdefs.search_expression = expression
self.kbdefs.collection = collection
db.session.merge(self.kbdefs)
else:
# insert
self.kbdefs = KnwKBDDEF(output_tag=field,
search_expression=expression,
collection=collection)
@staticmethod
def generate_slug(name):
"""Generate a slug for the knowledge.
:param name: text to slugify
:return: slugified text
"""
slug = slugify(name)
i = KnwKB.query.filter(
db.or_(
KnwKB.slug.like(slug),
KnwKB.slug.like(slug + '-%'),
)).count()
return slug + ('-{0}'.format(i) if i > 0 else '')
@staticmethod
def exists(kb_name):
"""Return True if a kb with the given name exists.
:param kb_name: the name of the knowledge base
:return: True if kb exists
"""
return KnwKB.query_exists(KnwKB.name.like(kb_name))
@staticmethod
def query_exists(filters):
"""Return True if a kb with the given filters exists.
E.g: KnwKB.query_exists(KnwKB.name.like('FAQ'))
:param filters: filter for sqlalchemy
:return: True if kb exists
"""
return db.session.query(KnwKB.query.filter(filters).exists()).scalar()
def get_filename(self):
"""Construct the file name for taxonomy knoledge."""
return cfg['CFG_WEBDIR'] + "/kbfiles/" \
+ str(self.id) + ".rdf"
class PersistentIdentifier(db.Model):
"""Store and register persistent identifiers.
Assumptions:
* Persistent identifiers can be represented as a string of max 255 chars.
* An object has many persistent identifiers.
* A persistent identifier has one and only one object.
"""
__tablename__ = 'pidSTORE'
__table_args__ = (
db.Index('uidx_type_pid', 'pid_type', 'pid_value', unique=True),
db.Index('idx_status', 'status'),
db.Index('idx_object', 'object_type', 'object_value'),
)
id = db.Column(db.Integer(15, unsigned=True), primary_key=True)
"""Id of persistent identifier entry."""
pid_type = db.Column(db.String(6), nullable=False)
"""Persistent Identifier Schema."""
pid_value = db.Column(db.String(length=255), nullable=False)
"""Persistent Identifier."""
pid_provider = db.Column(db.String(length=255), nullable=False)
"""Persistent Identifier Provider"""
status = db.Column(db.CHAR(length=1), nullable=False)
"""Status of persistent identifier, e.g. registered, reserved, deleted."""
object_type = db.Column(db.String(3), nullable=True)
"""Object Type - e.g. rec for record."""
object_value = db.Column(db.String(length=255), nullable=True)
"""Object ID - e.g. a record id."""
created = db.Column(db.DateTime(), nullable=False, default=datetime.now)
"""Creation datetime of entry."""
last_modified = db.Column(
db.DateTime(), nullable=False, default=datetime.now,
onupdate=datetime.now
)
"""Last modification datetime of entry."""
#
# Class methods
#
@classmethod
def create(cls, pid_type, pid_value, pid_provider='', provider=None):
"""Internally reserve a new persistent identifier.
A provider for the given persistent identifier type must exists. By
default the system will choose a provider according to the pid
type. If desired, the default system provider can be overridden via
the provider keyword argument.
Return PID object if successful otherwise None.
"""
# Ensure provider exists
if provider is None:
provider = PidProvider.create(pid_type, pid_value, pid_provider)
if not provider:
raise Exception(
"No provider found for %s:%s (%s)" % (
pid_type, pid_value, pid_provider)
)
db.session.begin_nested()
try:
obj = cls(pid_type=provider.pid_type,
pid_value=provider.create_new_pid(pid_value),
pid_provider=pid_provider,
status=cfg['PIDSTORE_STATUS_NEW'])
obj._provider = provider
db.session.add(obj)
obj.log("CREATE", "Created")
return obj
except SQLAlchemyError:
db.session.rollback()
obj.log("CREATE", "Failed to created. Already exists.")
return None
@classmethod
def get(cls, pid_type, pid_value, pid_provider='', provider=None):
"""Get persistent identifier.
Return None if not found.
"""
pid_value = to_unicode(pid_value)
obj = cls.query.filter_by(
pid_type=pid_type, pid_value=pid_value, pid_provider=pid_provider
).first()
if obj:
obj._provider = provider
return obj
else:
return None
#
# Instance methods
#
def has_object(self, object_type, object_value):
"""Determine if this PID is assigned to a specific object."""
if object_type not in cfg['PIDSTORE_OBJECT_TYPES']:
raise Exception("Invalid object type %s." % object_type)
object_value = to_unicode(object_value)
return self.object_type == object_type and \
self.object_value == object_value
def get_provider(self):
"""Get the provider for this type of persistent identifier."""
if self._provider is None:
self._provider = PidProvider.create(
self.pid_type, self.pid_value, self.pid_provider
)
return self._provider
def assign(self, object_type, object_value, overwrite=False):
"""Assign this persistent identifier to a given object.
Note, the persistent identifier must first have been reserved. Also,
if an exsiting object is already assigned to the pid, it will raise an
exception unless overwrite=True.
"""
if object_type not in cfg['PIDSTORE_OBJECT_TYPES']:
raise Exception("Invalid object type %s." % object_type)
object_value = to_unicode(object_value)
if not self.id:
raise Exception(
"You must first create the persistent identifier before you "
"can assign objects to it."
)
if self.is_deleted():
raise Exception(
"You cannot assign objects to a deleted persistent identifier."
)
with db.session.begin_nested():
# Check for an existing object assigned to this pid
existing_obj_id = self.get_assigned_object(object_type)
if existing_obj_id and existing_obj_id != object_value:
if not overwrite:
raise Exception(
"Persistent identifier is already assigned to another "
"object"
)
else:
self.log(
"ASSIGN",
"Unassigned object %s:%s (overwrite requested)" % (
self.object_type, self.object_value)
)
self.object_type = None
self.object_value = None
elif existing_obj_id and existing_obj_id == object_value:
# The object is already assigned to this pid.
return True
self.object_type = object_type
self.object_value = object_value
db.session.commit()
self.log("ASSIGN", "Assigned object {0}:{1}".format(
self.object_type, self.object_value
))
return True
def update(self, with_deleted=False, *args, **kwargs):
"""Update the persistent identifier with the provider."""
if self.is_new() or self.is_reserved():
raise Exception(
"Persistent identifier has not yet been registered."
)
if not with_deleted and self.is_deleted():
raise Exception("Persistent identifier has been deleted.")
with db.session.begin_nested():
provider = self.get_provider()
if provider is None:
self.log("UPDATE", "No provider found.")
raise Exception("No provider found.")
if provider.update(self, *args, **kwargs):
if with_deleted and self.is_deleted():
self.status = cfg['PIDSTORE_ST):TUS_REGISTERED']
return True
return False
def reserve(self, *args, **kwargs):
"""Reserve the persistent identifier with the provider.
Note, the reserve method may be called multiple times, even if it was
already reserved.
"""
if not (self.is_new() or self.is_reserved()):
raise Exception(
"Persistent identifier has already been registered."
)
with db.session.begin_nested():
provider = self.get_provider()
if provider is None:
self.log("RESERVE", "No provider found.")
raise Exception("No provider found.")
if provider.reserve(self, *args, **kwargs):
self.status = cfg['PIDSTORE_STATUS_RESERVED']
return True
return False
def register(self, *args, **kwargs):
"""Register the persistent identifier with the provider."""
if self.is_registered() or self.is_deleted():
raise Exception(
"Persistent identifier has already been registered."
)
with db.session.begin_nested():
provider = self.get_provider()
if provider is None:
self.log("REGISTER", "No provider found.")
raise Exception("No provider found.")
if provider.register(self, *args, **kwargs):
self.status = cfg['PIDSTORE_STATUS_REGISTERED']
return True
return False
def delete(self, *args, **kwargs):
"""Delete the persistent identifier."""
with db.session.begin_nested():
if self.is_new():
# New persistent identifier which haven't been registered yet.
# Just delete it completely but keep log)
# Remove links to log entries (leave the otherwise)
PidLog.query.filter_by(id_pid=self.id).update({'id_pid': None})
db.session.delete(self)
self.log("DELETE", "Unregistered PID successfully deleted")
else:
provider = self.get_provider()
if not provider.delete(self, *args, **kwargs):
return False
self.status = cfg['PIDSTORE_STATUS_DELETED']
return True
def sync_status(self, *args, **kwargs):
"""Synchronize persistent identifier status.
Used when the provider uses an external service, which might have been
modified outside of our system.
"""
with db.session.begin_nested():
provider = self.get_provider()
result = provider.sync_status(self, *args, **kwargs)
return result
def get_assigned_object(self, object_type=None):
"""Return an assigned object."""
if object_type is not None and self.object_type == object_type:
return self.object_value
return None
def is_registered(self):
"""Return true if the persistent identifier has been registered."""
return self.status == cfg['PIDSTORE_STATUS_REGISTERED']
def is_deleted(self):
"""Return true if the persistent identifier has been deleted."""
return self.status == cfg['PIDSTORE_STATUS_DELETED']
def is_new(self):
"""Return true if the PIDhas not yet been registered or reserved."""
return self.status == cfg['PIDSTORE_STATUS_NEW']
def is_reserved(self):
"""Return true if the PID has not yet been reserved."""
return self.status == cfg['PIDSTORE_STATUS_RESERVED']
def log(self, action, message):
"""Store action and message in log."""
if self.pid_type and self.pid_value:
message = "[%s:%s] %s" % (self.pid_type, self.pid_value, message)
with db.session.begin_nested():
p = PidLog(id_pid=self.id, action=action, message=message)
db.session.add(p)
class CheckerRuleExecution(db.Model):
__tablename__ = 'checker_rule_execution'
uuid = db.Column(
db.String(36),
primary_key=True,
doc="UUID of the execution. Same with that of RedisMaster and logfile.",
)
owner_id = db.Column(
db.Integer(15, unsigned=True),
db.ForeignKey('user.id'),
nullable=False,
default=1,
)
owner = db.relationship(
'User',
doc="User who owns this execution. May be used by reporters.",
)
rule_name = db.Column(
db.String(127),
db.ForeignKey('checker_rule.name'),
nullable=False,
index=True,
doc="Name of the associated task.",
)
_status = db.Column(
ChoiceType(StatusMaster, impl=db.Integer()),
default=StatusMaster.unknown,
)
status_update_date = db.Column(
db.DateTime(),
nullable=False,
server_default='1900-01-01 00:00:00',
doc="Last date the status was updated.",
)
start_date = db.Column(
db.DateTime(),
nullable=False,
server_default='1900-01-01 00:00:00',
doc="Date at which this task was started.",
)
dry_run = db.Column(
db.Boolean,
default=False,
doc=
"Whether this execution is a dry run. Note the `should_*` properties.")
@db.hybrid_property
def should_commit(self):
"""Whether this execution should commit record modifications."""
return not self.dry_run
@db.hybrid_property
def should_report_logs(self):
"""Whether this execution should report logs to the reporters."""
return not self.dry_run
@db.hybrid_property
def should_report_exceptions(self):
"""Whether this execution should report exceptions to the reporters."""
return not self.dry_run
@db.hybrid_property
def master(self):
"""The master object of this execution.
:rtype: `invenio_checker.clients.master.RedisMaster`
"""
return RedisMaster(self.uuid)
@db.hybrid_property
def status(self):
"""The status of the execution.
:rtype: `StatusMaster`
"""
return self._status
@status.setter
@session_manager
def status(self, new_status):
"""Status setter.
:type new_status: `StatusMaster`
"""
self._status = new_status
self.status_update_date = datetime.now()
def read_logs(self):
"""Stream user-friendly structured logs of this execution.
First attempt to stream using `eliot-tree` which provides a text
tree-like structure of the execution given its logs.
If `eliot-tree` fails (which happens when there is an eliot Task
serialization bug in our code, a warning is yielded, followed by the
output of `eliot-prettyprint`.
Therefore, the output of this function is not guaranteed to be
machine-readable.
..note::
This function may be called mid-run.
"""
from glob import glob
import subprocess
from .config import get_eliot_log_file
filenames = glob(get_eliot_log_file(master_id=self.uuid).name + "*")
eliottree_subp = subprocess.Popen(['eliot-tree', '--field-limit', '0'],
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
eliottree_failed = False
with eliottree_subp.stdin:
try:
for filename in filenames:
with open(filename, 'r') as file_:
eliottree_subp.stdin.write(file_.read())
except (IOError, MemoryError):
eliottree_failed = True
with eliottree_subp.stdout:
for line in eliottree_subp.stdout:
yield line
if eliottree_failed or (eliottree_subp.wait() != 0):
# eliot-tree can fail on unfinished logging. We still want output
# for debugging, so we use the less structured eliot-prettyprint
from eliot.prettyprint import pretty_format
from eliot._bytesjson import loads
yield '\n`eliot-tree` failed to format output. ' \
'Retrying with eliot-prettyprint:\n'
for filename in filenames:
yield "{}:\n".format(filename)
with open(filename, 'r') as file_:
for line in file_:
yield pretty_format(loads(line))
