def site_configuration_last_update(cls,
_db,
known_value=None,
timeout=None):
"""Check when the site configuration was last updated.
Updates Configuration.instance[Configuration.SITE_CONFIGURATION_LAST_UPDATE].
It's the application's responsibility to periodically check
this value and reload the configuration if appropriate.
:param known_value: We know when the site configuration was
last updated--it's this timestamp. Use it instead of checking
with the database.
:param timeout: We will only call out to the database once in
this number of seconds. If we are asked again before this
number of seconds elapses, we will assume site configuration
has not changed.
:return: a datetime object.
"""
now = datetime.datetime.utcnow()
if _db and timeout is None:
from model import ConfigurationSetting
timeout = ConfigurationSetting.sitewide(
_db, cls.SITE_CONFIGURATION_TIMEOUT).value
if timeout is None:
timeout = 60
last_check = cls.instance.get(
cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE)
if (not known_value and last_check
and (now - last_check).total_seconds() < timeout):
# We went to the database less than [timeout] seconds ago.
# Assume there has been no change.
return cls._site_configuration_last_update()
# Ask the database when was the last time the site
# configuration changed. Specifically, this is the last time
# site_configuration_was_changed() (defined in model.py) was
# called.
if not known_value:
from model import Timestamp
known_value = Timestamp.value(_db, cls.SITE_CONFIGURATION_CHANGED,
None)
if not known_value:
# The site configuration has never changed.
last_update = None
else:
last_update = known_value
# Update the Configuration object's record of the last update time.
cls.instance[cls.SITE_CONFIGURATION_LAST_UPDATE] = last_update
# Whether that record changed or not, the time at which we
# _checked_ is going to be set to the current time.
cls.instance[cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE] = now
return last_update
class Slide(object):
def __init__(self):
self.index = None
self.begin = Timestamp(Timestamp.SRT)
self.end = Timestamp(Timestamp.SRT)
self.content = []
@classmethod
def from_node(cls, node):
slide = cls()
slide.index = node['index']
slide.begin.millisecond = node['begin']
slide.end.millisecond = node['end']
slide.content = node['content']
return slide
@property
def valid(self):
return self.begin.millisecond > 0 and self.end.millisecond > 0 and \
self.begin.millisecond < self.end.millisecond and self.content
@property
def node(self):
return {
'index':self.index,
'begin':self.begin.millisecond,
'end':self.end.millisecond,
'content':self.content,
}
@property
def duration(self):
return self.end.millisecond - self.begin.millisecond
def clear(self):
self.content = []
def add(self, line):
if line:
line = line.strip()
if line:
self.content.append(line)
def shift(self, offset):
self.begin.shift(offset)
self.end.shift(offset)
def scale(self, factor):
self.begin.scale(factor)
self.end.scale(factor)
def encode(self, content):
content.append(unicode(self.index))
content.append(u'{0} --> {1}'.format(self.begin.timecode, self.end.timecode))
for line in self.content:
content.append(line)
content.append(u'')
def run_once_and_update_timestamp(self):
# First cover items that have never had a coverage attempt
# before.
offset = 0
while offset is not None:
offset = self.run_once(
offset, count_as_covered=BaseCoverageRecord.ALL_STATUSES)
# Next, cover items that failed with a transient failure
# on a previous attempt.
offset = 0
while offset is not None:
offset = self.run_once(
offset,
count_as_covered=BaseCoverageRecord.DEFAULT_COUNT_AS_COVERED)
Timestamp.stamp(self._db, self.service_name)
self._db.commit()
def do_run(self):
existing_timestamp = get_one(self._db, Timestamp, service=self.name)
if existing_timestamp:
raise Exception(
"Timestamp for Database Migration script already exists")
migrations = self.fetch_migration_files()[0]
most_recent_migration = self.sort_migrations(migrations)[-1]
initial_timestamp = Timestamp.stamp(self._db, self.name)
self.update_timestamp(initial_timestamp, most_recent_migration)
def setup(self):
super(TestDatabaseMigrationScript, self).setup()
self.script = MockDatabaseMigrationScript(_db=self._db)
# This list holds any temporary files created during tests
# so they can be deleted during teardown().
self.migration_files = []
self._create_test_migrations()
stamp = datetime.datetime.strptime('20260810', '%Y%m%d')
self.timestamp = Timestamp(service=self.script.name, timestamp=stamp)
self._db.add(self.timestamp)
def test_all(self):
"""Test that we can create a list of Monitors using all()."""
class OPDSCollectionMonitor(CollectionMonitor):
SERVICE_NAME = "Test Monitor"
PROTOCOL = ExternalIntegration.OPDS_IMPORT
# Here we have three OPDS import Collections...
o1 = self._collection()
o2 = self._collection()
o3 = self._collection()
# ...and a Bibliotheca collection.
b1 = self._collection(protocol=ExternalIntegration.BIBLIOTHECA)
# o1 just had its Monitor run.
Timestamp.stamp(self._db, OPDSCollectionMonitor.SERVICE_NAME, o1)
# o2 and b1 have never had their Monitor run, but o2 has had some other Monitor run.
Timestamp.stamp(self._db, "A Different Service", o2)
# o3 had its Monitor run an hour ago.
now = datetime.datetime.utcnow()
an_hour_ago = now - datetime.timedelta(seconds=3600)
Timestamp.stamp(self._db, OPDSCollectionMonitor.SERVICE_NAME, o3,
an_hour_ago)
monitors = list(OPDSCollectionMonitor.all(self._db))
# Three OPDSCollectionMonitors were returned, one for each
# appropriate collection. The monitor that needs to be run the
# worst was returned first in the list. The monitor that was
# run most recently is returned last. There is no
# OPDSCollectionMonitor for the Bibliotheca collection.
eq_([o2, o3, o1], [x.collection for x in monitors])
def test_run_starts_at_previous_counter(self):
# Two Identifiers.
i1, i2 = [self._identifier() for i in range(2)]
# The monitor was just run, but it was not able to proceed past
# i1.
timestamp = Timestamp.stamp(self._db, self.monitor.service_name,
self.monitor.collection)
timestamp.counter = i1.id
# Run the monitor.
self.monitor.run()
# The last item in the table was processed. i1 was not
# processed, because it was processed in a previous run.
eq_([i2], self.monitor.processed)
# The monitor's counter has been reset.
eq_(0, timestamp.counter)
def site_configuration_last_update(cls, _db, known_value=None,
timeout=None):
"""Check when the site configuration was last updated.
Updates Configuration.instance[Configuration.SITE_CONFIGURATION_LAST_UPDATE].
It's the application's responsibility to periodically check
this value and reload the configuration if appropriate.
:param known_value: We know when the site configuration was
last updated--it's this timestamp. Use it instead of checking
with the database.
:param timeout: We will only call out to the database once in
this number of seconds. If we are asked again before this
number of seconds elapses, we will assume site configuration
has not changed.
:return: a datetime object.
"""
now = datetime.datetime.utcnow()
if _db and timeout is None:
from model import ConfigurationSetting
timeout = ConfigurationSetting.sitewide(
_db, cls.SITE_CONFIGURATION_TIMEOUT
).value
if timeout is None:
timeout = 60
last_check = cls.instance.get(
cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE
)
if (not known_value
and last_check and (now - last_check).total_seconds() < timeout):
# We went to the database less than [timeout] seconds ago.
# Assume there has been no change.
return cls._site_configuration_last_update()
# Ask the database when was the last time the site
# configuration changed. Specifically, this is the last time
# site_configuration_was_changed() (defined in model.py) was
# called.
if not known_value:
from model import Timestamp
known_value = Timestamp.value(
_db, cls.SITE_CONFIGURATION_CHANGED, service_type=None,
collection=None
)
if not known_value:
# The site configuration has never changed.
last_update = None
else:
last_update = known_value
# Update the Configuration object's record of the last update time.
cls.instance[cls.SITE_CONFIGURATION_LAST_UPDATE] = last_update
# Whether that record changed or not, the time at which we
# _checked_ is going to be set to the current time.
cls.instance[cls.LAST_CHECKED_FOR_SITE_CONFIGURATION_UPDATE] = now
return last_update
def __init__(self):
self.index = None
self.time = Timestamp(Timestamp.CHAPTER)
self._name = None
self.language = None
class Chapter(object):
def __init__(self):
self.index = None
self.time = Timestamp(Timestamp.CHAPTER)
self._name = None
self.language = None
@classmethod
def from_raw(cls, timecode, name, format):
o = cls()
codec = Chapter.format[format]
if timecode and name:
# Decode timecode
match = codec["timecode decode"].search(timecode)
if match:
frag = match.groupdict()
o.time.timecode = o.time.codec["encode"].format(
int(frag["hour"]), int(frag["minute"]), int(frag["second"]), int(frag["millisecond"])
)
# Decode name
if codec["name decode"]:
match = codec["name decode"].search(name)
if match:
frag = match.groupdict()
o.name = frag["name"].replace(""", '"')
# if 'lang' in frag and frag['lang']:
# lang = self.env.enumeration['language'].parse(frag['lang'])
# if lang: self.language = lang
else:
o.name = name
return o
@classmethod
def from_node(cls, node):
o = cls()
o.index = node["index"]
o.time.millisecond = node["time"]
o.name = node["name"]
return o
@property
def valid(self):
return self.time is not None and self.time.millisecond > 0
@property
def name(self):
if self._name is None:
self._name = Chapter.default_name_format.format(self.index)
return self._name
@property
def node(self):
return {"index": self.index, "time": self.time.millisecond, "name": self.name}
@name.setter
def name(self, value):
match = Chapter.junk_name.search(value)
if match is None:
self._name = value.strip('"').strip("'").strip()
else:
self._name = None
def shift(self, offset):
self.time.shift(offset)
def scale(self, factor):
self.time.scale(factor)
def encode(self, content, format):
codec = Chapter.format[format]
content.append(codec["timecode encode"].format(self.index, self.time.timecode))
content.append(codec["name encode"].format(self.index, self.name))
def __unicode__(self):
return u"{}. {}:{}".format(self.index, self.time.timecode, self.name)
default_name_format = u"Chapter {0}"
junk_name = re.compile(ur"^(?:[0-9]{,2}:[0-9]{,2}:[0-9]{,2}[\.,][0-9]+|[0-9]+|chapter[\s0-9]+)$", re.UNICODE)
OGG = 1
MEDIAINFO = 2
format = {
1: {
"timecode encode": u"CHAPTER{0:02d}={1}",
"timecode decode": re.compile(
ur"CHAPTER(?P<index>[0-9]{,2})=(?P<hour>[0-9]{,2}):(?P<minute>[0-9]{,2}):(?P<second>[0-9]{,2})\.(?P<millisecond>[0-9]+)",
re.UNICODE,
),
"name encode": u"CHAPTER{0:02d}NAME={1}",
"name decode": re.compile(ur"CHAPTER(?P<index>[0-9]{,2})NAME=(?P<name>.*)", re.UNICODE),
},
2: {
"timecode encode": None,
"timecode decode": re.compile(
ur"_(?P<hour>[0-9]{,2})_(?P<minute>[0-9]{,2})_(?P<second>[0-9]{2})(?P<millisecond>[0-9]{3})", re.UNICODE
),
"name encode": None,
"name decode": re.compile(ur"(?:(?P<lang>[a-z]{2}):)?(?P<name>.*)", re.UNICODE),
},
}
def test_error_not_raised_when_timestamp_forced(self):
Timestamp.stamp(self._db, self.script.name)
self.script.do_run(['-f'])
self.assert_matches_latest_migration()
def test_error_raised_when_timestamp_exists(self):
Timestamp.stamp(self._db, self.script.name)
assert_raises(RuntimeError, self.script.do_run)
def __init__(self):
self.index = None
self.begin = Timestamp(Timestamp.SRT)
self.end = Timestamp(Timestamp.SRT)
self.content = []
