def test_using_transactions(self):
with Using(slave1, [A]) as txn:
list(B.select())
A.create(data='a1')
B.create(data='b1')
self.assertDatabaseVerb([
('slave1', 'BEGIN'),
('master', 'SELECT'),
('slave1', 'INSERT'),
('master', 'INSERT')])
def fail_with_exc(data):
with Using(slave2, [A]):
A.create(data=data)
raise ValueError('xxx')
self.assertRaises(ValueError, fail_with_exc, 'a2')
self.assertDatabaseVerb([
('slave2', 'BEGIN'),
('slave2', 'INSERT')])
with Using(slave1, [A, B]):
a_objs = [a_obj.data for a_obj in A.select()]
self.assertEqual(a_objs, ['a1'])
def test_using_context(self):
models = [A, B]
with Using(slave1, models, False):
A.create(data='a1')
B.create(data='b1')
self.assertDatabaseVerb([
('slave1', 'INSERT'),
('slave1', 'INSERT')])
with Using(slave2, models, False):
A.create(data='a2')
B.create(data='b2')
a_obj = A.select().order_by(A.id).first()
self.assertEqual(a_obj.data, 'a1')
self.assertDatabaseVerb([
('slave2', 'INSERT'),
('slave2', 'INSERT'),
('slave2', 'SELECT')])
with Using(master, models, False):
query = A.select().order_by(A.data.desc())
values = [a_obj.data for a_obj in query]
self.assertEqual(values, ['a2', 'a1'])
self.assertDatabaseVerb([('master', 'SELECT')])
def wrapper(*args, **kwargs):
language = kwargs.get("language", "en")
path = kwargs.pop("database_path", None)
if not path:
path = CONTENT_DATABASE_PATH.format(channel=kwargs.get(
"channel", CHANNEL),
language=language)
db = SqliteDatabase(path, pragmas=settings.CONTENT_DB_SQLITE_PRAGMAS)
kwargs["db"] = db
db.connect()
# This should contain all models in the database to make them available to the wrapped function
with Using(db, [Item, AssessmentItem]):
try:
output = function(*args, **kwargs)
except DoesNotExist:
output = None
except OperationalError as e:
logging.error(
"Content DB error: Perhaps content database file found? "
"Exception: {e}".format(e=str(e)))
raise
db.close()
return output
def get_medias_series(self):
with Using(self.database, [SeriesMedias, SeriesSubtitles],
with_transaction=False):
limit = 100
medias = (SeriesMedias.select().order_by(
SeriesMedias.series_title.desc()).limit(limit).execute())
return [{
"added_timestamp":
med.added_timestamp.strftime('%Y-%m-%dT%H:%M:%S'),
"series_title":
med.series_title,
"season_number":
med.season_number,
"episode_number":
med.episode_number,
"episode_title":
med.episode_title,
"quality":
med.quality,
"video_languages":
med.video_languages,
"subtitle_languages":
sorted(s.language for s in med.subtitles),
"dirty":
med.dirty,
} for med in medias]
def setup_content_paths(context, db):
"""
Creaters available content items and adds their urls to the context object.
:param context: A behave context, to which the attributes "available_content_path" and "unavailable_content_path"
will be added.
:return: None
"""
# These paths are "magic" -- the success or failure of actually visiting the content items in the browser
# depends on these specific values.
context.unavailable_content_path, context.available_content_path = (
"khan/foo/bar/unavail",
"khan/math/arithmetic/addition-subtraction/basic_addition/addition_1/",
)
# This function uses 'iterator_content_items' function to return a list of path, update dict pairs
# It then updates the items with these paths with their update dicts, and then propagates
# availability changes up the topic tree - this means that we can alter the availability of one item
# and make all its parent topics available so that it is navigable to in integration tests.
def iterator_content_items(ids=None, channel="khan", language="en"):
return [(context.available_content_path, {"available": True})]
annotate_content_models(db=db,
iterator_content_items=iterator_content_items)
with Using(db, [Item], with_transaction=False):
context._unavailable_item = Item.create(
title="Unavailable item",
description="baz",
available=False,
kind="Video",
id="3",
slug="unavail",
path=context.unavailable_content_path)
def setUp(self, db=None):
self.db = db
with Using(db, [Item], with_transaction=False):
parent = self.parent = Item.create(
title="Foo",
description="Bar",
available=True,
kind="Topic",
id="1",
slug="foo",
path="foopath"
)
self.available_item = Item.create(
title="available_item",
description="Bingo",
available=True,
kind="Topic",
id="2",
slug="avail",
path="avail",
parent=parent
)
self.unavailable_item = Item.create(
title="Unavailable item",
description="baz",
available=False,
kind="Topic",
id="3",
slug="unavail",
path="unavail",
parent=parent
)
def test_update_content_availability_true(self):
with Using(self.db, [Item]):
actual = dict(
update_content_availability([
unparse_model_data(model_to_dict(self.item))
])).get("thepath")
assert actual.get("available")
def get_recent_events(self, limit: int):
with Using(self.database, [Events], with_transaction=False):
events = (Events.select().limit(limit).order_by(
Events.timestamp.desc()).execute())
return [{
"timestamp": e.timestamp.strftime('%Y-%m-%dT%H:%M:%S'),
"type": e.type,
"message": e.message
} for e in events]
def teardown_content_paths(context, db):
"""
The opposite of ``setup_content_urls``. Removes content items created there.
:param context: A behave context, which keeps a reference to the Items so we can clean them up.
:return: None.
"""
with Using(db, [Item], with_transaction=False):
context._unavailable_item.delete_instance()
def tearDown(self):
with Using(self.db, [Item], with_transaction=False):
self.item.delete_instance()
if self.cleanup:
try:
os.remove(self.version_path)
except OSError:
pass
def recurse_availability_up_tree(nodes, db) -> [Item]:
logging.info("Marking availability.")
nodes = list(nodes)
def _recurse_availability_up_tree(node):
available = node.available
if not node.parent:
return node
else:
parent = node.parent
Parent = Item.alias()
children = Item.select().join(
Parent,
on=(Item.parent == Parent.pk)).where(Item.parent == parent.pk)
if not available:
children_available = children.where(
Item.available == True).count() > 0
available = children_available
total_files = children.aggregate(fn.SUM(Item.total_files))
child_remote = children.where((
(Item.available == False) & (Item.kind != "Topic"))
| (Item.kind == "Topic")).aggregate(
fn.SUM(Item.remote_size))
child_on_disk = children.aggregate(fn.SUM(Item.size_on_disk))
if parent.available != available:
parent.available = available
if parent.total_files != total_files:
parent.total_files = total_files
# Ensure that the aggregate sizes are not None
if parent.remote_size != child_remote and child_remote:
parent.remote_size = child_remote
# Ensure that the aggregate sizes are not None
if parent.size_on_disk != child_on_disk and child_on_disk:
parent.size_on_disk = child_on_disk
if parent.is_dirty():
parent.save()
_recurse_availability_up_tree(parent)
return node
with Using(db, [Item]):
# at this point, the only thing that can affect a topic's availability
# are exercises. Videos and other content's availability can only be
# determined by what's in the client. However, we need to set total_files
# and remote_sizes. So, loop over exercises and other content,
# and skip topics as they will be recursed upwards.
for node in (n for n in nodes if n.kind != NodeType.topic):
_recurse_availability_up_tree(node)
return nodes
def teardown_content_db(instance, db):
"""
Seems to split out in a classmethod because BDD base_environment wants
to reuse it.
"""
with Using(db, [Item], with_transaction=False):
instance.content_unavailable_item.delete_instance()
instance.content_root.delete_instance()
for item in (instance.content_exercises + instance.content_videos +
instance.content_subsubtopics +
instance.content_subtopics):
item.delete_instance()
def save_models(nodes, db):
"""
Save all the models in nodes into the db specified.
"""
# aron: I didn't bother writing tests for this, since it's such a simple
# function!
db.create_table(Item, safe=True)
with Using(db, [Item]):
for node in nodes:
try:
node.save()
except Exception as e:
logging.warning("Cannot save {path}, exception: {e}".format(path=node.path, e=e))
yield node
def save_assessment_items(assessment_items, db):
"""
Save all the models in nodes into the db specified.
"""
# aron: I didn't bother writing tests for this, since it's such a simple
# function!
db.create_table(AssessmentItem, safe=True)
with Using(db, [AssessmentItem]):
for item in assessment_items:
try:
item.save()
except Exception as e:
logging.warning("Cannot save {id}, exception: {e}".format(id=item.id, e=e))
yield item
def test_writes_db_to_archive(self):
with tempfile.NamedTemporaryFile() as zffobj:
zf = zipfile.ZipFile(zffobj, "w")
with tempfile.NamedTemporaryFile() as dbfobj:
db = SqliteDatabase(dbfobj.name)
db.connect()
with Using(db, [Item]):
Item.create_table()
item = Item(id="test",
title="test",
description="test",
available=False,
slug="srug",
kind=NodeType.video,
path="/test/test")
item.save()
db.close()
save_db(db, zf)
zf.close()
# reopen the db from the zip, see if our object was saved
with tempfile.NamedTemporaryFile() as f:
# we should only have one file in the zipfile, the db. Assume
# that the first file is the db.
zf = zipfile.ZipFile(zffobj.name)
dbfobj = zf.open(zf.infolist()[0])
f.write(dbfobj.read())
f.seek(0)
db = SqliteDatabase(f.name)
with Using(db, [Item]):
Item.get(title="test")
def test_update_content_availability_false(self):
try:
os.rename(self.version_path, self.version_path + ".bak")
except OSError:
pass
with Using(self.db, [Item]):
actual = dict(
update_content_availability([
unparse_model_data(model_to_dict(self.item))
])).get("thepath")
# Update is only generated if changed from False to True, not from False to False, so should return None.
assert not actual
try:
os.rename(self.version_path + ".bak", self.version_path)
except OSError:
pass
def update_fetched_series_subtitles(self,
series_episode_uid,
subtitles_languages,
dirty=True):
with Using(self.database, [SeriesMedias, SeriesSubtitles],
with_transaction=False):
media = (SeriesMedias.select().where(
SeriesMedias.tv_db_id == series_episode_uid.tv_db_id,
SeriesMedias.season_number == series_episode_uid.season_number,
SeriesMedias.episode_number ==
series_episode_uid.episode_number))
for lang in subtitles_languages:
self._get_or_create(
SeriesSubtitles,
series_media=media,
language=lang,
)
for m in media:
m.dirty = dirty
m.save()
def get_last_fetched_series(self, limit: int):
with Using(self.database, [SeriesMedias, SeriesSubtitles],
with_transaction=False):
events = (SeriesSubtitles.select().order_by(
SeriesSubtitles.added_timestamp.desc()).limit(limit).execute())
return [{
"added_timestamp":
e.added_timestamp.strftime('%Y-%m-%dT%H:%M:%S'),
"series_title":
e.series_media.series_title,
"season_number":
e.series_media.season_number,
"episode_number":
e.series_media.episode_number,
"episode_title":
e.series_media.episode_title,
"quality":
e.series_media.quality,
"video_languages":
e.series_media.video_languages,
"subtitle_language":
e.language,
} for e in events]
def setUp(self, db=None):
self.db = db
with Using(db, [Item], with_transaction=False):
self.item = Item(
title=self.TITLE,
available=self.AVAILABLE,
kind=self.KIND,
description="test",
id="counting-out-1-20-objects",
slug="test",
path="thepath",
extra_fields={},
)
self.item.save()
self.version_path = contentload_settings.KHAN_ASSESSMENT_ITEM_VERSION_PATH
self.cleanup = False
if not os.path.exists(self.version_path):
with open(self.version_path, 'w') as f:
f.write("stuff")
self.cleanup = True
def update_series_media(self,
series_title,
tv_db_id,
season_number,
episode_number,
episode_title,
quality,
video_languages,
media_filename,
dirty=True):
assert media_filename, "media_filename cannot be None"
with Using(self.database, [SeriesMedias], with_transaction=False):
media, _ = self._get_or_create(SeriesMedias,
tv_db_id=tv_db_id,
season_number=season_number,
episode_number=episode_number,
media_filename=media_filename)
media.series_title = series_title
media.episode_title = episode_title
media.quality = quality
media.video_languages = video_languages
media.dirty = dirty
media.media_filename = media_filename
media.save()
def fail_with_exc(data):
with Using(slave2, [A]):
A.create(data=data)
raise ValueError('xxx')
def media_exists(self, media_filename):
with Using(self.database, [SeriesMedias], with_transaction=False):
medias = (SeriesMedias.select(SeriesMedias.media_filename).where(
SeriesMedias.media_filename == media_filename)).execute()
return bool(medias)
def insert_event(self, thetype: str, message: str):
with Using(self.database, [Events], with_transaction=False):
Events.create(type=thetype, message=message)
def tearDown(self):
with Using(self.db, [Item], with_transaction=False):
self.available_item.delete_instance()
self.unavailable_item.delete_instance()
self.parent.delete_instance()
def setup_content_db(instance, db):
# Setup the content.db (defaults to the en version)
with Using(db, [Item], with_transaction=False):
# Root node
instance.content_root = Item.create(title="Khan Academy",
description="",
available=True,
files_complete=0,
total_files="1",
kind="Topic",
parent=None,
id="khan",
slug="khan",
path="khan/",
extra_fields="{}",
youtube_id=None,
remote_size=315846064333,
sort_order=0)
for _i in range(4):
slug = "topic{}".format(_i)
instance.content_subtopics.append(
Item.create(
title="Subtopic {}".format(_i),
description="A subtopic",
available=True,
files_complete=0,
total_files="4",
kind="Topic",
parent=instance.content_root,
id=slug,
slug=slug,
path="khan/{}/".format(slug),
extra_fields="{}",
remote_size=1,
sort_order=_i,
))
# Parts of the content recommendation system currently is hard-coded
# to look for 3rd level recommendations only and so will fail if we
# don't have this level of lookup
for subtopic in instance.content_subtopics:
for _i in range(4):
slug = "{}-{}".format(subtopic.id, _i)
instance.content_subsubtopics.append(
Item.create(
title="{} Subsubtopic {}".format(subtopic.title, _i),
description="A subsubtopic",
available=True,
files_complete=4,
total_files="4",
kind="Topic",
parent=subtopic,
id=slug,
slug=slug,
path="{}{}/".format(subtopic.path, slug),
youtube_id=None,
extra_fields="{}",
remote_size=1,
sort_order=_i,
))
# We need at least 10 exercises in some of the tests to generate enough
# data etc.
# ...and we need at least some exercises in each sub-subtopic
for parent in instance.content_subsubtopics:
# Make former created exercise the prerequisite of the next one
prerequisite = None
for _i in range(4):
slug = "{}-exercise-{}".format(parent.id, _i)
extra_fields = {}
if prerequisite:
extra_fields['prerequisites'] = [prerequisite.id]
new_exercise = Item.create(
title="Exercise {} in {}".format(_i, parent.title),
parent=parent,
description="Solve this",
available=True,
kind="Exercise",
id=slug,
slug=slug,
path="{}{}/".format(parent.path, slug),
sort_order=_i,
**extra_fields)
instance.content_exercises.append(new_exercise)
prerequisite = new_exercise
# Add some videos, too, even though files don't exist
for parent in instance.content_subsubtopics:
for _i in range(4):
slug = "{}-video-{}".format(parent.pk, _i)
instance.content_videos.append(
Item.create(
title="Video {} in {}".format(_i, parent.title),
parent=random.choice(instance.content_subsubtopics),
description="Watch this",
available=True,
kind="Video",
id=slug,
slug=slug,
path="{}{}/".format(parent.path, slug),
extra_fields={
"subtitle_urls": [],
"content_urls": {
"stream": "/foo",
"stream_type": "video/mp4"
},
},
sort_order=_i))
with Using(db, [Item], with_transaction=False):
instance.content_unavailable_item = Item.create(
title="Unavailable item",
description="baz",
available=False,
kind="Video",
id="unavail123",
slug="unavail",
path=instance.content_unavailable_content_path,
parent=random.choice(instance.content_subsubtopics).pk,
)
instance.content_available_content_path = random.choice(
instance.content_exercises).path