def set_content(self, content, interpret=True): if self.binary_interpreter is not None and interpret: content = self.binary_interpreter.decipher(content) else: if type(content) is not bytes: raise Exception("Bytes are required as content.") if content == self.cached_content: return False if self.smart_updater is not None: self.smart_updater.queue_content_update( "datasets/{}/elements/content".format( self.dataset_owner.get_url_prefix()), self.get_id(), content) else: upload_pool.submit(self._upload_content, content) self.cached_content = content self.has_content = True self.cached_content_time = now() if self.content_promise: self.content_promise = None return True
def __init__(self, title, description, tags, http_ref, id=None, dataset_owner=None, token=None, binary_interpreter=None, token_info=None, server_info=None, smart_updater: AsyncSmartUpdater = None, api_url=None): super().__init__(token, token_info=token_info, server_info=server_info, api_url=api_url) self.data = { 'title': title, 'description': description, 'tags': tags, 'http_ref': http_ref } self.has_content = False self.dataset_owner = dataset_owner self.binary_interpreter = binary_interpreter self.token = token self._id = id self.cached_content = None self.cached_content_time = now() self.content_promise = None self.smart_updater = smart_updater
def get_content(self, interpret=True): if not self.has_content: return None if self.content_promise is not None: self.cached_content = self.content_promise.result()[self._id] self.cached_content_time = now() self.content_promise = None if (now() - self.cached_content_time).total_seconds() > CACHE_TIME: self.cached_content = None if self.cached_content is None: content = self._retrieve_content()[self._id] else: content = self.cached_content if self.binary_interpreter is not None and interpret: content = self.binary_interpreter.cipher(content) return content
def _get_key(self, key_index, options=None): ps = int(self.server_info['Page-Size']) key_page = key_index // ps index = key_index % ps dumped_options = json.dumps(options) if json.dumps(options) not in self.page_cache: self.page_cache[dumped_options] = {} try: if (now() - self.last_cache_time).total_seconds() > CACHE_TIME: self.page_cache[dumped_options].clear() page = self.page_cache[dumped_options][key_page] except KeyError as ex: # Cache miss page = self._get_json("datasets/{}/elements".format(self.get_url_prefix()), extra_data={'page': key_page}, json_data={'options': options}) self.page_cache[dumped_options][key_page] = page self.last_cache_time = now() return page[index]['_id']
def __init__(self, url_prefix: str, title: str, description: str, reference: str, tags: list, token: str=None, binary_interpreter: Interpreter=None, token_info: dict=None, server_info: dict=None, use_smart_updater: AsyncSmartUpdater=True, owner=None, api_url=None): self.data = {} self.binary_interpreter = binary_interpreter """:type : Interpreter""" if "/" in url_prefix: self.data['url_prefix'] = url_prefix else: if token is not None: token_prefix = token.get_prefix() self.data['url_prefix'] = '{}/{}'.format(token_prefix, url_prefix) self.data['title'] = title self.data['description'] = description self.data['tags'] = tags self.data['reference'] = reference self.data['size'] = 0 self.elements_count = 0 self.comments_count = 0 self.page_cache = {} self.last_cache_time = now() self.owner = owner super().__init__(token, token_info=token_info, server_info=server_info, api_url=api_url) # Server_info is only available after super() init. if use_smart_updater: self.smart_updater = AsyncSmartUpdater(self.server_info, self) """ :type : AsyncSmartUpdater """ else: self.smart_updater = None """