def cache_file_at_url(self, url):
""" Caches the remote file or retrieves its cached version if one exists. Returns None if an error occurs. """
filename, extension = os.path.splitext(url)
cache_asset_hash = hash(str(self.channel_id) + filename)
asset = Asset.selectBy(plugin_channel=self.channel_id,
filename=filename,
is_cached=True).getOne(None)
if asset is None:
if CacheManager._get_lock(cache_asset_hash, blocking=False):
try:
asset = Asset(plugin_channel=self.channel_id,
filename=filename,
user=None,
extension=extension if extension is not None
and len(extension) > 0 else None,
is_cached=True)
self.download_manager.enqueue_asset(asset)
except (URLError, OSError):
logger.warning(
'Exception encountered when attempting to cache file at url %s',
url,
exc_info=True)
CacheManager._release_lock(cache_asset_hash)
return None
CacheManager._release_lock(cache_asset_hash)
else:
CacheManager._get_lock(cache_asset_hash, blocking=True)
CacheManager._release_lock(cache_asset_hash)
asset = Asset.selectBy(plugin_channel=self.channel_id,
filename=filename,
is_cached=True).getOne(None)
return asset
def cleanup_cache(self):
""" Cleans up the cached assets by deleting all assets not referenced during this day. """
today = datetime.date.today()
unused_assets = Asset.selectBy(is_cached=True).filter(Asset.q.last_reference < sqlbuilder.func.date(str(today)))
total_assets_size = 0
if unused_assets.count() > 0:
total_assets_size = int(unused_assets.sum(Asset.q.file_size))
Asset.deleteMany(AND(Asset.q.last_reference < sqlbuilder.func.date(str(today)), Asset.q.is_cached == True))
logger.info('Ran cache cleanup and deleted %d assets for a total size of %s', unused_assets.count(),
CleanupScheduler._human_readable_size(total_assets_size))
next_cleanup = datetime.datetime.combine(today + datetime.timedelta(days=1), datetime.time(hour=23, minute=55))
self.s.enterabs(time.mktime(next_cleanup.timetuple()), 1, self.cleanup_cache)
def get_cached_file(self, filename):
"""
Returns an Asset if this cache manager has a cached version of the given file for this channel
or None if none exists.
"""
return Asset.selectBy(plugin_channel=self.channel_id,
filename=filename,
is_cached=True).getOne(None)
def contains_video(self):
for field, inputs in self.content.items():
if 'file' in inputs:
if Asset.get(inputs['file']).mime_type.startswith('video'):
return True
elif 'video' in inputs:
return True
return False
def create_database():
Building.createTable()
Channel.createTable()
Plugin.createTable()
User.createTable()
PluginChannel.createTable()
ChannelBundle.createTable()
Role.createTable()
Screen.createTable()
ScreenMac.createTable()
Subscription.createTable()
Template.createTable()
Asset.createTable()
PluginParamAccessRights.createTable()
LogStat.createTable()
DBVersion.createTable()
DBVersion(version=database_version)
User(username="******", fullname="ICTV Admin", email="admin@ictv", super_admin=True, disabled=False)
def __init__(self, slide_input, template):
if 'video' not in slide_input and 'file' not in slide_input:
raise ValueError('No video was specified in either video or file slide input type')
self.content = {'background-1': slide_input}
if 'background-1' not in Templates[template]:
raise KeyError('The template %s has no background to put the video' % template)
video_path = slide_input['video'] if 'video' in slide_input else Asset.get(slide_input['file']).path
video_info = MediaInfo.parse(video_path)
self.duration = video_info.tracks[0].duration + 1000
self.template = template
def runTest(self):
""" Tests the cache mecanism. """
u = User(username='******',
fullname='testasset test',
email='*****@*****.**',
super_admin=True,
disabled=False)
PluginChannel(name='testasset2',
plugin=Plugin(name='cache_plugin', activated='no'),
subscription_right='restricted')
c = PluginChannel.selectBy(name="testasset2").getOne()
a = Asset(plugin_channel=c, user=u)
self.testApp.get('/cache/' + str(a.id), status=303)
try:
Asset.delete(a.id)
except SQLObjectNotFound:
pass
finally:
User.delete(u.id)
PluginChannel.delete(c.id)
def get(self, asset_id):
""" Waits for the given asset to be downloaded. Redirects the user to the asset when done. """
try:
asset_id = int(asset_id)
if self.download_manager.has_pending_task_for_asset(asset_id):
task = self.download_manager.get_pending_task_for_asset(asset_id)
task.result()
resp.seeother('/' + Asset.get(asset_id)._get_path(force=True)) # Task is complete but asset may be still marked as in flight
except SQLObjectNotFound:
resp.notfound()
except KeyError:
resp.seeother('/cache/' + str(asset_id))
def render_page(self):
channels = []
total_size = 0
for channel in Asset.select().throughTo.plugin_channel.distinct():
channel_info = {'id': channel.id, 'name': channel.name, 'plugin': channel.plugin.name, 'assets_count': channel.assets.count(),
'assets_size': channel.assets.sum(Asset.q.file_size) or 0, 'cache_percentage': 0}
cache_size = channel.assets.filter(Asset.q.is_cached == True).sum(Asset.q.file_size)
if cache_size is not None:
channel_info['cache_percentage'] = cache_size / channel_info['assets_size']
total_size += (channel_info['assets_size'] if channel_info['assets_size'] is not None else 0)
channels.append(channel_info)
for channel_info in channels:
channel_info['percentage'] = ((channel_info['assets_size'] / total_size) if channel_info['assets_size'] and total_size is not None else 0)
return self.renderer.storage(channels=channels)
def _is_video(file_ref):
asset = Asset.get(file_ref)
return asset.mime_type.startswith('video')
def runTest(self):
""" Tests the Asset SQLObject. """
fake_plugin = Plugin(name='fake_plugin', activated='notfound')
asset_channel = PluginChannel(name='Asset Channel',
plugin=fake_plugin,
subscription_right='public')
a1 = Asset(plugin_channel=asset_channel,
user=None,
filename='path_test',
extension='.txt')
last_ref_a1 = a1.last_reference
time.sleep(1)
assert a1.path == os.path.join('static', 'storage',
str(asset_channel.id),
str(a1.id) + '.txt')
assert a1.last_reference > last_ref_a1
a2 = Asset(plugin_channel=asset_channel,
user=None,
filename='path_test')
assert a2.path == os.path.join('static', 'storage',
str(asset_channel.id), str(a2.id))
a3 = Asset(plugin_channel=asset_channel,
user=None,
filename='cache_test',
in_flight=True)
a3_path = os.path.join('static', 'storage', str(asset_channel.id),
str(a3.id))
assert a3.path is None
assert a3._get_path(force=True) == a3_path
a3.in_flight = False
assert a3.path == a3_path
a4 = Asset(plugin_channel=asset_channel,
user=None,
filename='test_write',
extension='.txt')
a4_path = os.path.join(get_root_path(), 'static', 'storage',
str(asset_channel.id),
str(a4.id) + '.txt')
a4_content = 'a4 file content'.encode()
a4.write_to_asset_file(a4_content)
with open(a4_path, 'rb') as f:
assert f.read() == a4_content
a4.destroySelf()
assert not os.path.exists(a4_path)
