def get(self, **kwargs): if DEVICE_SETTINGS_CACHE_KEY not in cache: model = super(DeviceSettingsManager, self).get(**kwargs) cache.set(DEVICE_SETTINGS_CACHE_KEY, model, 600) else: model = cache.get(DEVICE_SETTINGS_CACHE_KEY) return model
def clear_channel_stats(channel_id): cached_keys = process_cache.get( CHANNEL_STATS_CACHED_KEYS.format(channel_id=channel_id), set()) for key in cached_keys: process_cache.delete(key) process_cache.set(CHANNEL_STATS_CACHED_KEYS.format(channel_id=channel_id), set(), None)
def get_available_checksums_from_disk(channel_id, drive_id): try: basepath = get_mounted_drive_by_id(drive_id).datafolder except KeyError: raise LocationError("Drive with id {} does not exist".format(drive_id)) PER_DISK_CACHE_KEY = "DISK_AVAILABLE_CHECKSUMS_{basepath}".format( basepath=basepath) PER_DISK_PER_CHANNEL_CACHE_KEY = ( "DISK_AVAILABLE_CHECKSUMS_{basepath}_{channel_id}".format( basepath=basepath, channel_id=channel_id)) if PER_DISK_PER_CHANNEL_CACHE_KEY not in process_cache: if PER_DISK_CACHE_KEY not in process_cache: content_dir = get_content_storage_dir_path(datafolder=basepath) disk_checksums = [] for _, _, files in os.walk(content_dir): for name in files: checksum = os.path.splitext(name)[0] # Only add valid checksums formatted according to our standard filename if checksum_regex.match(checksum): disk_checksums.append(checksum) # Cache is per device, so a relatively long lived one should # be fine. process_cache.set(PER_DISK_CACHE_KEY, disk_checksums, 3600) else: disk_checksums = process_cache.get(PER_DISK_CACHE_KEY) checksums = set( LocalFile.objects.filter( files__contentnode__channel_id=channel_id).values_list( "id", flat=True)).intersection(set(disk_checksums)) process_cache.set(PER_DISK_PER_CHANNEL_CACHE_KEY, checksums, 3600) else: checksums = process_cache.get(PER_DISK_PER_CHANNEL_CACHE_KEY) return checksums
def get_app_key(cls): key = cache.get(APP_KEY_CACHE_KEY) if key is None: try: app_key = cls.objects.get() except cls.DoesNotExist: app_key = cls.update_app_key() key = app_key.key cache.set(APP_KEY_CACHE_KEY, key, 5000) return key
def get_cache_key(cls): key = cache.get(CONTENT_CACHE_KEY_CACHE_KEY) if key is None: try: cache_key = cls.objects.get() except cls.DoesNotExist: cache_key = cls.update_cache_key() key = cache_key.key cache.set(CONTENT_CACHE_KEY_CACHE_KEY, key, 5000) return key
def get_channel_stats_from_studio(channel_id): CACHE_KEY = "STUDIO_CHANNEL_STATS_{channel_id}".format( channel_id=channel_id) if CACHE_KEY not in process_cache: channel_stats = get_channel_annotation_stats(channel_id) process_cache.set(CACHE_KEY, channel_stats, 3600) register_key_as_cached(CACHE_KEY, channel_id) else: channel_stats = process_cache.get(CACHE_KEY) return channel_stats
def get_channel_stats_from_disk(channel_id, drive_id): CACHE_KEY = "DISK_CHANNEL_STATS_{drive_id}_{channel_id}".format( drive_id=drive_id, channel_id=channel_id) if CACHE_KEY not in process_cache: checksums = get_available_checksums_from_disk(channel_id, drive_id) channel_stats = get_channel_annotation_stats(channel_id, checksums) process_cache.set(CACHE_KEY, channel_stats, 3600) register_key_as_cached(CACHE_KEY, channel_id) else: channel_stats = process_cache.get(CACHE_KEY) return channel_stats
def get_channel_stats_from_peer(channel_id, peer_id): CACHE_KEY = "PEER_CHANNEL_STATS_{peer_id}_{channel_id}".format( peer_id=peer_id, channel_id=channel_id) if CACHE_KEY not in process_cache: checksums = get_available_checksums_from_remote(channel_id, peer_id) channel_stats = get_channel_annotation_stats(channel_id, checksums) process_cache.set(CACHE_KEY, channel_stats, 3600) register_key_as_cached(CACHE_KEY, channel_id) else: channel_stats = process_cache.get(CACHE_KEY) return channel_stats
def device_port_open(self): """ check to see if a port is open at a given `base_url` """ cached = process_cache.get( DEVICE_PORT_CACHE_KEY.format(url=self.base_url)) if cached: return cached result = check_if_port_open(self.base_url) process_cache.set(DEVICE_PORT_CACHE_KEY.format(url=self.base_url), result, DEVICE_PORT_TIMEOUT) return result
def get_available_checksums_from_remote(channel_id, peer_id): """ The current implementation prioritizes minimising requests to the remote server. In order to achieve this, it caches based on the baseurl and the channel_id. Also, it POSTs the complete list of non-supplementary files to the rmeote endpoint, and thus can keep this representation cached regardless of how the availability on the local server has changed in the interim. """ try: baseurl = NetworkLocation.objects.values_list( "base_url", flat=True).get(id=peer_id) except NetworkLocation.DoesNotExist: raise LocationError("Peer with id {} does not exist".format(peer_id)) CACHE_KEY = "PEER_AVAILABLE_CHECKSUMS_{baseurl}_{channel_id}".format( baseurl=baseurl, channel_id=channel_id) if CACHE_KEY not in process_cache: channel_checksums = (LocalFile.objects.filter( files__contentnode__channel_id=channel_id, files__supplementary=False).values_list("id", flat=True).distinct()) response = requests.post( get_file_checksums_url(channel_id, baseurl), data=compress_string( bytes(json.dumps(list(channel_checksums)).encode("utf-8"))), headers={"content-type": "application/gzip"}, ) checksums = None # Do something if we got a successful return if response.status_code == 200: try: integer_mask = int(response.content) # Filter to avoid passing in bad checksums checksums = set( compress(channel_checksums, _generate_mask_from_integer(integer_mask))) process_cache.set(CACHE_KEY, checksums, 3600) except (ValueError, TypeError): # Bad JSON parsing will throw ValueError # If the result of the json.loads is not iterable, a TypeError will be thrown # If we end up here, just set checksums to None to allow us to cleanly continue pass else: checksums = process_cache.get(CACHE_KEY) return checksums
def check_device_info(self): from kolibri.core.discovery.models import NetworkLocation info = check_device_info(self.base_url) if info: # Update the underlying model at the same time as the cache NetworkLocation.objects.filter( instance_id=info.get("instance_id")).update(**info) process_cache.set( DEVICE_INFO_CACHE_KEY.format(url=self.base_url), info, DEVICE_INFO_TIMEOUT, ) return info
def update_app_key(cls): app_key, created = cls.objects.get_or_create() app_key.key = uuid4().hex app_key.save() cache.set(APP_KEY_CACHE_KEY, app_key.key, 5000) return app_key
def update_cache_key(cls): cache_key, created = cls.objects.get_or_create() cache_key.key = time.time() cache_key.save() cache.set(CONTENT_CACHE_KEY_CACHE_KEY, cache_key.key, 5000) return cache_key
def save(self, *args, **kwargs): self.pk = 1 self.full_clean() super(DeviceSettings, self).save(*args, **kwargs) cache.set(DEVICE_SETTINGS_CACHE_KEY, self, 600)
def set_device_port_cache(url, is_open): process_cache.set(DEVICE_PORT_CACHE_KEY.format(url=url), is_open)
def set_device_info_cache(url, info): process_cache.set(DEVICE_INFO_CACHE_KEY.format(url=url), info)
def diff_stats(channel_id, method, drive_id=None, baseurl=None): """ Download the channel database to an upgraded path. Annotate the local file availability of the upgraded channel db. Calculate diff stats comparing default db and annotated channel db. """ # upgraded content database path source_path = get_upgrade_content_database_file_path(channel_id) # annotated db to be used for calculating diff stats destination_path = get_annotated_content_database_file_path(channel_id) try: if method == "network": call_command("importchannel", "network", channel_id, baseurl=baseurl, no_upgrade=True) elif method == "disk": drive = get_mounted_drive_by_id(drive_id) call_command("importchannel", "disk", channel_id, drive.datafolder, no_upgrade=True) # create all fields/tables at the annotated destination db, based on the current schema version bridge = Bridge(sqlite_file_path=destination_path, schema_version=CURRENT_SCHEMA_VERSION) bridge.Base.metadata.create_all(bridge.engine) # initialize import manager based on annotated destination path, pulling from source db path import_manager = channel_import.initialize_import_manager( channel_id, cancel_check=False, source=source_path, destination=destination_path, ) # import channel data from source db path import_manager.import_channel_data() import_manager.end() # annotate file availability on destination db annotation.set_local_file_availability_from_disk( destination=destination_path) # get the diff count between whats on the default db and the annotated db ( new_resource_ids, new_resource_content_ids, new_resource_total_size, ) = get_new_resources_available_for_import(destination_path, channel_id) # get the count for leaf nodes which are in the default db, but not in the annotated db resources_to_be_deleted_count = count_removed_resources( destination_path, channel_id) # get the ids of leaf nodes which are now incomplete due to missing local files ( updated_resource_ids, updated_resource_content_ids, updated_resource_total_size, ) = get_automatically_updated_resources(destination_path, channel_id) # remove the annotated database try: os.remove(destination_path) except OSError as e: logger.info( "Tried to remove {}, but exception {} occurred.".format( destination_path, e)) # annotate job metadata with diff stats job = get_current_job() if job: job.extra_metadata["new_resources_count"] = len( new_resource_content_ids) job.extra_metadata[ "deleted_resources_count"] = resources_to_be_deleted_count job.extra_metadata["updated_resources_count"] = len( updated_resource_content_ids) job.save_meta() CACHE_KEY = CHANNEL_UPDATE_STATS_CACHE_KEY.format(channel_id) process_cache.set( CACHE_KEY, { "new_resource_ids": new_resource_ids, "new_resource_content_ids": new_resource_content_ids, "new_resource_total_size": new_resource_total_size, "updated_resource_ids": updated_resource_ids, "updated_resource_content_ids": updated_resource_content_ids, "updated_resource_total_size": updated_resource_total_size, }, # Should persist until explicitly cleared (at content import) # or until server restart. None, ) except UserCancelledError: # remove the annotated database try: os.remove(destination_path) except OSError: pass raise
def register_key_as_cached(key, channel_id): cached_keys = process_cache.get( CHANNEL_STATS_CACHED_KEYS.format(channel_id=channel_id), set()) cached_keys.add(key) process_cache.set(CHANNEL_STATS_CACHED_KEYS.format(channel_id=channel_id), cached_keys, None)