def get(self, **kwargs):
if DEVICE_SETTINGS_CACHE_KEY not in cache:
model = super(DeviceSettingsManager, self).get(**kwargs)
cache.set(DEVICE_SETTINGS_CACHE_KEY, model, 600)
else:
model = cache.get(DEVICE_SETTINGS_CACHE_KEY)
return model
def clear_channel_stats(channel_id):
cached_keys = process_cache.get(
CHANNEL_STATS_CACHED_KEYS.format(channel_id=channel_id), set())
for key in cached_keys:
process_cache.delete(key)
process_cache.set(CHANNEL_STATS_CACHED_KEYS.format(channel_id=channel_id),
set(), None)
def get_available_checksums_from_disk(channel_id, drive_id):
try:
basepath = get_mounted_drive_by_id(drive_id).datafolder
except KeyError:
raise LocationError("Drive with id {} does not exist".format(drive_id))
PER_DISK_CACHE_KEY = "DISK_AVAILABLE_CHECKSUMS_{basepath}".format(
basepath=basepath)
PER_DISK_PER_CHANNEL_CACHE_KEY = (
"DISK_AVAILABLE_CHECKSUMS_{basepath}_{channel_id}".format(
basepath=basepath, channel_id=channel_id))
if PER_DISK_PER_CHANNEL_CACHE_KEY not in process_cache:
if PER_DISK_CACHE_KEY not in process_cache:
content_dir = get_content_storage_dir_path(datafolder=basepath)
disk_checksums = []
for _, _, files in os.walk(content_dir):
for name in files:
checksum = os.path.splitext(name)[0]
# Only add valid checksums formatted according to our standard filename
if checksum_regex.match(checksum):
disk_checksums.append(checksum)
# Cache is per device, so a relatively long lived one should
# be fine.
process_cache.set(PER_DISK_CACHE_KEY, disk_checksums, 3600)
else:
disk_checksums = process_cache.get(PER_DISK_CACHE_KEY)
checksums = set(
LocalFile.objects.filter(
files__contentnode__channel_id=channel_id).values_list(
"id", flat=True)).intersection(set(disk_checksums))
process_cache.set(PER_DISK_PER_CHANNEL_CACHE_KEY, checksums, 3600)
else:
checksums = process_cache.get(PER_DISK_PER_CHANNEL_CACHE_KEY)
return checksums
def get_app_key(cls):
key = cache.get(APP_KEY_CACHE_KEY)
if key is None:
try:
app_key = cls.objects.get()
except cls.DoesNotExist:
app_key = cls.update_app_key()
key = app_key.key
cache.set(APP_KEY_CACHE_KEY, key, 5000)
return key
def get_cache_key(cls):
key = cache.get(CONTENT_CACHE_KEY_CACHE_KEY)
if key is None:
try:
cache_key = cls.objects.get()
except cls.DoesNotExist:
cache_key = cls.update_cache_key()
key = cache_key.key
cache.set(CONTENT_CACHE_KEY_CACHE_KEY, key, 5000)
return key
def get_channel_stats_from_studio(channel_id):
CACHE_KEY = "STUDIO_CHANNEL_STATS_{channel_id}".format(
channel_id=channel_id)
if CACHE_KEY not in process_cache:
channel_stats = get_channel_annotation_stats(channel_id)
process_cache.set(CACHE_KEY, channel_stats, 3600)
register_key_as_cached(CACHE_KEY, channel_id)
else:
channel_stats = process_cache.get(CACHE_KEY)
return channel_stats
def get_channel_stats_from_disk(channel_id, drive_id):
CACHE_KEY = "DISK_CHANNEL_STATS_{drive_id}_{channel_id}".format(
drive_id=drive_id, channel_id=channel_id)
if CACHE_KEY not in process_cache:
checksums = get_available_checksums_from_disk(channel_id, drive_id)
channel_stats = get_channel_annotation_stats(channel_id, checksums)
process_cache.set(CACHE_KEY, channel_stats, 3600)
register_key_as_cached(CACHE_KEY, channel_id)
else:
channel_stats = process_cache.get(CACHE_KEY)
return channel_stats
def get_channel_stats_from_peer(channel_id, peer_id):
CACHE_KEY = "PEER_CHANNEL_STATS_{peer_id}_{channel_id}".format(
peer_id=peer_id, channel_id=channel_id)
if CACHE_KEY not in process_cache:
checksums = get_available_checksums_from_remote(channel_id, peer_id)
channel_stats = get_channel_annotation_stats(channel_id, checksums)
process_cache.set(CACHE_KEY, channel_stats, 3600)
register_key_as_cached(CACHE_KEY, channel_id)
else:
channel_stats = process_cache.get(CACHE_KEY)
return channel_stats
def device_port_open(self):
""" check to see if a port is open at a given `base_url` """
cached = process_cache.get(
DEVICE_PORT_CACHE_KEY.format(url=self.base_url))
if cached:
return cached
result = check_if_port_open(self.base_url)
process_cache.set(DEVICE_PORT_CACHE_KEY.format(url=self.base_url),
result, DEVICE_PORT_TIMEOUT)
return result
def get_available_checksums_from_remote(channel_id, peer_id):
"""
The current implementation prioritizes minimising requests to the remote server.
In order to achieve this, it caches based on the baseurl and the channel_id.
Also, it POSTs the complete list of non-supplementary files to the rmeote endpoint,
and thus can keep this representation cached regardless of how the availability on
the local server has changed in the interim.
"""
try:
baseurl = NetworkLocation.objects.values_list(
"base_url", flat=True).get(id=peer_id)
except NetworkLocation.DoesNotExist:
raise LocationError("Peer with id {} does not exist".format(peer_id))
CACHE_KEY = "PEER_AVAILABLE_CHECKSUMS_{baseurl}_{channel_id}".format(
baseurl=baseurl, channel_id=channel_id)
if CACHE_KEY not in process_cache:
channel_checksums = (LocalFile.objects.filter(
files__contentnode__channel_id=channel_id,
files__supplementary=False).values_list("id",
flat=True).distinct())
response = requests.post(
get_file_checksums_url(channel_id, baseurl),
data=compress_string(
bytes(json.dumps(list(channel_checksums)).encode("utf-8"))),
headers={"content-type": "application/gzip"},
)
checksums = None
# Do something if we got a successful return
if response.status_code == 200:
try:
integer_mask = int(response.content)
# Filter to avoid passing in bad checksums
checksums = set(
compress(channel_checksums,
_generate_mask_from_integer(integer_mask)))
process_cache.set(CACHE_KEY, checksums, 3600)
except (ValueError, TypeError):
# Bad JSON parsing will throw ValueError
# If the result of the json.loads is not iterable, a TypeError will be thrown
# If we end up here, just set checksums to None to allow us to cleanly continue
pass
else:
checksums = process_cache.get(CACHE_KEY)
return checksums
def check_device_info(self):
from kolibri.core.discovery.models import NetworkLocation
info = check_device_info(self.base_url)
if info:
# Update the underlying model at the same time as the cache
NetworkLocation.objects.filter(
instance_id=info.get("instance_id")).update(**info)
process_cache.set(
DEVICE_INFO_CACHE_KEY.format(url=self.base_url),
info,
DEVICE_INFO_TIMEOUT,
)
return info
def update_app_key(cls):
app_key, created = cls.objects.get_or_create()
app_key.key = uuid4().hex
app_key.save()
cache.set(APP_KEY_CACHE_KEY, app_key.key, 5000)
return app_key
def update_cache_key(cls):
cache_key, created = cls.objects.get_or_create()
cache_key.key = time.time()
cache_key.save()
cache.set(CONTENT_CACHE_KEY_CACHE_KEY, cache_key.key, 5000)
return cache_key
def save(self, *args, **kwargs):
self.pk = 1
self.full_clean()
super(DeviceSettings, self).save(*args, **kwargs)
cache.set(DEVICE_SETTINGS_CACHE_KEY, self, 600)
def set_device_port_cache(url, is_open):
process_cache.set(DEVICE_PORT_CACHE_KEY.format(url=url), is_open)
def set_device_info_cache(url, info):
process_cache.set(DEVICE_INFO_CACHE_KEY.format(url=url), info)
def diff_stats(channel_id, method, drive_id=None, baseurl=None):
"""
Download the channel database to an upgraded path.
Annotate the local file availability of the upgraded channel db.
Calculate diff stats comparing default db and annotated channel db.
"""
# upgraded content database path
source_path = get_upgrade_content_database_file_path(channel_id)
# annotated db to be used for calculating diff stats
destination_path = get_annotated_content_database_file_path(channel_id)
try:
if method == "network":
call_command("importchannel",
"network",
channel_id,
baseurl=baseurl,
no_upgrade=True)
elif method == "disk":
drive = get_mounted_drive_by_id(drive_id)
call_command("importchannel",
"disk",
channel_id,
drive.datafolder,
no_upgrade=True)
# create all fields/tables at the annotated destination db, based on the current schema version
bridge = Bridge(sqlite_file_path=destination_path,
schema_version=CURRENT_SCHEMA_VERSION)
bridge.Base.metadata.create_all(bridge.engine)
# initialize import manager based on annotated destination path, pulling from source db path
import_manager = channel_import.initialize_import_manager(
channel_id,
cancel_check=False,
source=source_path,
destination=destination_path,
)
# import channel data from source db path
import_manager.import_channel_data()
import_manager.end()
# annotate file availability on destination db
annotation.set_local_file_availability_from_disk(
destination=destination_path)
# get the diff count between whats on the default db and the annotated db
(
new_resource_ids,
new_resource_content_ids,
new_resource_total_size,
) = get_new_resources_available_for_import(destination_path,
channel_id)
# get the count for leaf nodes which are in the default db, but not in the annotated db
resources_to_be_deleted_count = count_removed_resources(
destination_path, channel_id)
# get the ids of leaf nodes which are now incomplete due to missing local files
(
updated_resource_ids,
updated_resource_content_ids,
updated_resource_total_size,
) = get_automatically_updated_resources(destination_path, channel_id)
# remove the annotated database
try:
os.remove(destination_path)
except OSError as e:
logger.info(
"Tried to remove {}, but exception {} occurred.".format(
destination_path, e))
# annotate job metadata with diff stats
job = get_current_job()
if job:
job.extra_metadata["new_resources_count"] = len(
new_resource_content_ids)
job.extra_metadata[
"deleted_resources_count"] = resources_to_be_deleted_count
job.extra_metadata["updated_resources_count"] = len(
updated_resource_content_ids)
job.save_meta()
CACHE_KEY = CHANNEL_UPDATE_STATS_CACHE_KEY.format(channel_id)
process_cache.set(
CACHE_KEY,
{
"new_resource_ids": new_resource_ids,
"new_resource_content_ids": new_resource_content_ids,
"new_resource_total_size": new_resource_total_size,
"updated_resource_ids": updated_resource_ids,
"updated_resource_content_ids": updated_resource_content_ids,
"updated_resource_total_size": updated_resource_total_size,
},
# Should persist until explicitly cleared (at content import)
# or until server restart.
None,
)
except UserCancelledError:
# remove the annotated database
try:
os.remove(destination_path)
except OSError:
pass
raise
def register_key_as_cached(key, channel_id):
cached_keys = process_cache.get(
CHANNEL_STATS_CACHED_KEYS.format(channel_id=channel_id), set())
cached_keys.add(key)
process_cache.set(CHANNEL_STATS_CACHED_KEYS.format(channel_id=channel_id),
cached_keys, None)
