def _remove_dir_contents(path, pattern=None): # os.path.lexists is not supported on windows from cocktail.styled import styled exists = getattr(os.path, "lexists", os.path.exists) if exists(path): if pattern: items = glob(os.path.join(path, pattern)) else: items = os.listdir(path) for item in items: item_path = os.path.join(path, item) if debug: print styled(" " * 4 + item_path, "red") try: if os.path.isdir(item_path): rmtree(item_path) else: os.remove(item_path) except OSError, ex: if debug: print styled(ex, "red")
def set_expiration(self, key: CacheKey, expiration: Optional[int]): """Sets the expiration assigned to the given key. :param key: The key to update the expiration for. :param expiration: The new expiration date for the key. Can be set to None to disable time based expiration for the key; otherwise, check the `~cocktail.caching.utils.normalize_expiration` function for all the possible ways of specifying the expiration date for the key. :raises cocktail.caching.CacheKeyError: Raised if the key is not contained in the storage or has already expired. :raises ValueError: Raised if the given expiration key is not valid (ie. is not an integer timestamp, or is set in the past). """ if not self.enabled or self.storage is None: raise CacheKeyError(key) norm_key = self.normalize_key(key) if self.verbose: print((styled("CACHE", "white", "dark_gray") + " " + styled("Set expiration", "pink", style="bold") + "\n" + styled(" Key:", "light_gray", style="bold") + " " + norm_key + "\n" + styled(" Expiration:", "light_gray", style="bold") + " " + str(expiration) + "\n")) if expiration is not None: expiration = normalize_expiration(expiration) self.storage.set_expiration(norm_key, expiration)
def clear(self, scope: Scope = whole_cache): """Discards all keys in the cache matching the given scope. :param scope: A selector indicating the keys that should be removed. It can take the following forms: - `~cocktail.caching.whole_cache` will remove all the entries in the cache. - A string will be treated as a single tag, and will delete all entries tagged with said tag. - A tuple of strings will be interpreted as an intersection of tags: any entry tagged with *all* of the tags in the tuple will be deleted. - Collections will be treated as a union of other selectors: all entries matching *any* of the selectors in the collection will be removed. :raises TypeError: Raised if the ``scope`` parameter is given a value of a wrong type. """ if self.enabled and self.storage is not None: if self.verbose: print( (styled("CACHE", "white", "dark_gray") + " " + styled("Clear", "magenta", style="bold") + "\n" + styled(" Scope:", "light_gray", style="bold") + " " + ("whole cache" if scope is whole_cache else repr(scope)) + "\n")) self.storage.clear(scope=normalize_scope(scope))
def discard(self, key: CacheKey) -> bool: """Removes the given key from the cache, if it exists. This method is similar to `.remove`, but it will return a boolean indicating wether the removal succeeded (`.remove` raises an exception if it can't find the requested key in the cache). :param key: The key to remove. :return: True if the key was removed from the cache, False if it wasn't contained on the cache, or if it had already expired. """ if not self.enabled or self.storage is None: return False norm_key = self.normalize_key(key) if self.storage.discard(norm_key): if self.verbose: print((styled("CACHE", "white", "dark_gray") + " " + styled("Remove", "magenta", style="bold") + "\n" + styled(" Key:", "light_gray", style="bold") + " " + norm_key + "\n")) return True else: return False
def print_memory_usage(self): if self.memory_limit: usage = self.memory_usage limit = self.memory_limit ratio = usage / limit percent = int(ratio * 100) bar = ( "%s / %s" % ( format_bytes(usage), format_bytes(limit) ) ).rjust(self.memory_usage_bar_width) for threshold, bar_color in self.memory_usage_colors: if percent >= threshold: break bar_width = int(ratio * self.memory_usage_bar_width) info = ( styled(bar[:bar_width], "white", bar_color) + styled(bar[bar_width:], "white", "dark_gray") ) else: info = format_bytes(self.memory_usage) print("Memory usage: %s\n" % info)
def _is_current(self, resource, invalidation=None, verbose=False): # Expiration if resource.has_expired(default_expiration=self.expiration): if verbose: print(styled("ResourceLoader: Resource expired", "white", "brown", "bold"), resource.key, end=' ') if resource.expiration is None: print(self.expiration) else: print(resource.expiration) return False # Invalidation if callable(invalidation): invalidation = invalidation() if invalidation is not None: if isinstance(invalidation, datetime): invalidation = mktime(invalidation.timetuple()) if invalidation > resource.creation: if verbose: print(styled("ResourceLoader: Resource invalidated", "white", "brown", "bold"), end=' ') print(resource.key, invalidation) return False return True
def step1(self, code=None, target_url=None, **kwargs): if target_url: self.target_url = target_url session[SESSION_PREFIX + "target_url"] = target_url flow = OAuth2WebServerFlow(self.provider.client_id, self.provider.client_secret, self.provider.scope, redirect_uri=self.step_url(1)) flow.params["access_type"] = self.provider.access_type if not code: redirect(flow.step1_get_authorize_url()) if self.provider.debug_mode: print(styled("Google authorization code:", "magenta"), code) credentials = flow.step2_exchange(code) session[SESSION_PREFIX + "credentials"] = credentials if self.provider.debug_mode: print(styled("Google refresh token:", "magenta"), end=' ') print(credentials.refresh_token) print(styled("Google access token:", "magenta"), end=' ') print(credentials.access_token) redirect(self.step_url(2))
def print_tree(self, indent=""): print(indent + styled(repr(self), "slate_blue")) indent += " " for group in self.groups: group.print_tree(indent) for item in self.items: print(indent + styled(item, "pink"))
def extract(self, member, value, language=None): if value is None: return False if self.verbose: print((" " * 4 * len(self.__stack)) + "Extracting", end=' ') print(styled(member, style="bold"), end=' ') print("from", end=' ') print(styled(value, style="bold"), end=' ') if language: print("in language", styled(language, style="bold")) else: print() is_object = isinstance(type(value), Schema) if is_object: if value in self.__visited: return False else: self.__visited.add(value) node = TextExtractorNode(member, language, value, [], self.__stack[-1] if self.__stack else None) self.__stack.append(node) try: member.extract_searchable_text(self) finally: self.__stack.pop(-1) return True
def _insert_into_manifest(e): obj = e.source if obj.synchronizable: global_id = e.source.global_id if global_id: get_manifest()[global_id] = None if debug: print styled("Manifest: declared " + obj.global_id, "bright_green")
def show_blame(self): for cls, tb_count in self.blame.iteritems(): print styled(cls, "slate_blue") for tb, count in tb_count.most_common(): print print "%s reads" % styled(count, style="bold") for fname, line, func, code in tb: print " %s %s:%d" % (func, fname, line) print " " + styled(code, "light_gray")
def _invalidate_manifest(e): from cocktail.styled import styled obj = e.source if not obj.is_inserted: return # Elegibility for synchronization changed if e.member is Item.synchronizable: # Object becomes synchronizable: add it to the manifest if e.value: if obj.global_id: get_manifest()[obj.global_id] = None if debug: print styled("Manifest: declared " + obj.global_id, "bright_green") # Object ceases to be synchronizable: remove it from the manifest else: get_manifest().pop(obj.global_id, None) if debug: print styled("Manifest: removed " + obj.global_id, "magenta") return elif not obj.synchronizable: return # Object global identifier changed if e.member is Item.global_id: manifest = get_manifest() if e.previous_value: manifest.pop(e.previous_value, None) if debug: print styled("Manifest: removed " + obj.global_id, "magenta") if e.value: manifest[e.value] = None if debug: print styled("Manifest: declared " + obj.global_id, "bright_green") return elif not obj.global_id: return # Changing a synchronizable member: invalidate the stored object's hash in # the manifest, if any if e.member.synchronizable: get_manifest()[obj.global_id] = None if debug: print styled("Manifest: invalidated " + obj.global_id, "cyan")
def set_value(self, key, value, expiration=None): if self.verbose: print(styled("ResourceLoader: Storing", "white", "pink", "bold"), end=' ') print(key, end=' ') if expiration is None: print() else: print(styled("expiration:", "pink"), expiration) self.__resources[key] = Resource(key, value, expiration)
def list_action(self): tasks = list(self.tasks) tasks.sort() tasks = defaultdict( (lambda: defaultdict( (lambda: defaultdict(list)) )) ) for action, publishable, language in self.tasks: tasks[action][publishable.__class__][publishable].append(language) for action in sorted(tasks): print() print( styled( action.upper().ljust(120), "white", "green" if action == "post" else "red", "bold" ) ) for cls, cls_tasks in tasks[action].iteritems(): cls_label = translations(cls) print() print( styled( cls_label.ljust(120), "white", "dark_gray", "bold" ) ) for publishable, languages in cls_tasks.iteritems(): print(str(publishable.id).ljust(10), end=" ") print( styled( translations( publishable, discard_generic_translation=True, language=languages[0] )[:36].ljust(38), "slate_blue" ), end=" " ) if languages != [None]: print(styled(", ".join(languages), "pink")) else: print()
def remove_links(file, links = None, encoding = None): if links is None: links = get_links(file) for link in links: link = encode_filename(link, encoding) if os.path.lexists(link): if debug: print styled("STATIC PUBLICATION", "white", "red"), print "Removing link for #%s (%s)" % ( styled(file.id, style = "bold"), styled(link, "yellow") ) os.remove(link)
def get_refresh_token(self, code, redirect_uri=None): """ Gets an authorization token with access_token and refresh_token from an authorization code. :param code: A valid authorization code with access_type offline :return: A dict representing the authorization token. The 'access_token' key is a valid access token and the 'refresh_token' key can be used to generate new valid access tokens """ TOKEN_URL = "https://accounts.google.com/o/oauth2/token" if not redirect_uri: website = (app.website or Configuration.instance.websites[0]) redirect_uri = "http://%s/google_oauth/%d/refresh_token" % ( website.hosts[0], self.id) params = { "code": code, "client_id": self.client_id, "client_secret": self.client_secret, "redirect_uri": redirect_uri, "grant_type": "authorization_code" } request = urllib.request.Request(url=TOKEN_URL, data=urllib.parse.urlencode(params)) json_file = urllib.request.urlopen(request).read() auth_token = json.loads(json_file) if self.debug_mode: print(styled("Google refresh token:", "magenta"), auth_token) return auth_token
def transaction(action, action_args=(), action_kwargs=None, max_attempts=3, before_retrying=None, desist=None): if action_kwargs is None: action_kwargs = {} for i in range(max_attempts): if i > 0: if verbose: print( styled( "Retrying transaction %s (%d/%d)" % (action, i, max_attempts - 1), { 1: "yellow", 2: "brown", 3: "red" }.get(i, "violet"))) if before_retrying is not None: before_retrying(*action_args, **action_kwargs) if desist is not None and desist(*action_args, **action_kwargs): return desisted try: rvalue = action(*action_args, **action_kwargs) datastore.commit() return rvalue except ConflictError: datastore.sync() # implicit abort raise
def store(self, key: CacheKey, value: Any, expiration: Optional[int] = None, tags: Optional[Set[str]] = None): """Inserts or updates a value in the storage. If the key already existed, the given value, expiration and tags will be used to replace the old ones. :param key: A string that will uniquely identify the value in the cache, making it possible to `.retrieve` it later. :param value: The value to associate to this key. :param expiration: An optional parameter that limits the maximum life span of the key in the storage. See the `~cocktail.caching.utils.normalize_expiration` function for a detailed explanation of all the forms the parameter can take. :param tags: An optional set of tags to attach to this key. If given, it should be expressed as a collection of strings, each one representing a single tag. """ if not self.enabled or self.storage is None: return norm_key = self.normalize_key(key) if self.verbose: lines = [ styled("CACHE", "white", "dark_gray") + " " + styled("Storing", "slate_blue", style="bold"), styled(" Key:", "light_gray", style="bold") + " " + norm_key ] if expiration is not None: lines.append( styled(" Expiration:", "light_gray", style="bold") + " " + str(expiration)) if tags is not None: lines.append( styled(" Tags:", "light_gray", style="bold") + " " + str(sorted(tags))) lines.append("") print("\n".join(lines)) if expiration is not None: expiration = normalize_expiration(expiration) self.storage.store(norm_key, value, expiration=expiration, tags=tags)
def clear_image_cache(item=None, factory=None): if not app.root: return if debug: from cocktail.styled import styled print styled("Clearing image cache", "red"), if item: print styled("Item:", "light_gray"), print styled(item, "red", style="bold"), if factory: print styled("Factory:", "light_gray"), print styled(factory, "red", style="bold"), print # Remove the full cache if item is None and factory is None: _remove_dir_contents(app.path("image-cache")) _remove_dir_contents(app.path("static", "images")) # Selective drop: per item and/or factory else: paths = [] if item is not None: paths.append(app.path("image-cache", str(item.id))) paths.append(app.path("static", "images", str(item.id))) else: for base in (app.path("image-cache"), app.path("static", "images")): for item in os.listdir(base): path = os.path.join(base, item) if os.path.isdir(path): paths.append(path) if factory is None: pattern = None else: pattern = factory + ".*" for path in paths: _remove_dir_contents(path, pattern)
def step1(self, code=None, target_url=None, **kwargs): self.check_step1_errors(**kwargs) if target_url: self.target_url = target_url session[SESSION_PREFIX + "target_url"] = target_url if not code: params = { 'client_id': self.provider.client_id, 'redirect_uri': self.step_url(1), 'scope': ','.join(self.provider.scope) } login_uri = 'https://www.facebook.com/dialog/oauth?' + \ urllib.parse.urlencode(params) redirect(login_uri) if self.provider.debug_mode: print(styled("Facebook authorization code:", "magenta"), code) params = { 'client_id': self.provider.client_id, 'redirect_uri': self.step_url(1), 'client_secret': self.provider.client_secret, 'code': code } token_uri = 'https://graph.facebook.com/v2.3/oauth/access_token?' \ + urllib.parse.urlencode(params) json_file = urllib.request.urlopen(token_uri).readline() token_data = json.loads(json_file) if not token_data.get("access_token"): raise FacebookOAuthBadResponseException( token_data, "Expected an 'access_token' key") session[SESSION_PREFIX + "credentials"] = token_data if self.provider.debug_mode: print(styled("Facebook token data:", "magenta"), end=' ') print(token_data) redirect(self.step_url(2))
def drop_weight(self) -> Optional[CacheKey]: """Removes an entry from the cache, in order to free resources. :return: The key that has been removed, or ``None`` if the cache is empty. """ if not self.enabled or self.storage is None: return key = self.storage.drop_weight() if key and self.verbose: print((styled("CACHE", "white", "dark_gray") + " " + styled("Drop weight", "magenta", style="bold") + "\n" + styled(" Key:", "light_gray", style="bold") + " " + key + "\n")) return key
def clear_after_commit(self, scope: Scope = whole_cache): """Schedules a cache invalidation operation after the current ZODB transaction is committed. This method can be called multiple times per transaction, with each successive call widening the scope that will be affected once the transaction is completed. That said, if the scope is set to `~cocktail.caching.whole_cache`, the scope indicated by past or future invocations will be disregarded, and the whole cache will be cleared once the transaction is completed. :param scope: Determines the keys that will be removed once the transaction is completed. See the `.clear` method for details on its accepted formats. """ if not self.enabled or self.storage is None or not scope: return key = (TRANSACTION_KEY, id(self)) from cocktail.persistence import datastore transaction_invalidation_scope = datastore.get_transaction_value(key) if transaction_invalidation_scope is whole_cache: return elif transaction_invalidation_scope is None: datastore.unique_after_commit_hook( TRANSACTION_KEY, _cache_invalidation_commit_handler, self) scope = normalize_scope(scope) if transaction_invalidation_scope is None: datastore.set_transaction_value(key, scope) elif scope is whole_cache: datastore.set_transaction_value(key, scope) else: transaction_invalidation_scope.update(scope) if self.verbose: print( (styled("CACHE", "white", "dark_gray") + " " + styled("Clear after commit", "magenta", style="bold") + "\n" + styled(" Scope:", "light_gray", style="bold") + " " + ("whole cache" if scope is whole_cache else repr(scope)) + "\n"))
def main(): params = argv[1:] if not params: print "Usage: %s filelist" % argv[0] exit(1) for param in params: for path in glob(param): print "Importing " + styled(path, style="bold") + "...", try: file = File.from_path( path, resource_filename("_PROJECT_NAME_", "upload")) file.insert() except Exception, ex: print styled(str(ex), "red") else: print styled("File #" + str(file.id), "green")
def clear_audio_cache(item=None, encoder_id=None): if debug: from cocktail.styled import styled print styled("Clearing audio cache", "red"), if item: print styled("Item:", "light_gray"), print styled(item, "red", style="bold"), if encoder_id: print styled("Encoder:", "light_gray"), print styled(encoder_id, "red", style="bold"), print # Remove the full cache if item is None and encoder_id is None: _remove_dir_contents(app.path("audio-cache")) _remove_dir_contents(app.path("static", "audio")) # Selective drop: per item and/or encoder else: paths = [] if item is not None: paths.append(app.path("audio-cache", str(item.id))) paths.append(app.path("static", "audio", str(item.id))) else: for base in (app.path("audio-cache"), app.path("static", "audio")): for item in os.listdir(base): path = os.path.join(base, item) if os.path.isdir(path): paths.append(path) if encoder_id is None: pattern = None else: pattern = encoder_id + ".*" for path in paths: _remove_dir_contents(path, pattern)
def retrieve_with_metadata(self, key: CacheKey) -> Tuple[Any, int, Set[str]]: """Obtains the value, expiration and tags for the given key. :param key: The key to retrieve. :return: A tuple containing the value, expiration and tags associated with the indicated key. :raises cocktail.caching.CacheKeyError: Raised if the key is not present in the `.storage` for the cache, or if it has expired. """ if not self.enabled or self.storage is None: raise CacheKeyError(key) norm_key = self.normalize_key(key) if self.verbose: try: value = self.storage.retrieve_with_metadata(norm_key) except CacheKeyError: print((styled("CACHE", "white", "dark_gray") + " " + styled("Miss", "red", style="bold") + "\n" + styled(" Key:", "light_gray", style="bold") + " " + norm_key + "\n")) raise else: print((styled("CACHE", "white", "dark_gray") + " " + styled("Hit", "bright_green", style="bold") + "\n" + styled(" Key:", "light_gray", style="bold") + " " + norm_key + "\n")) return value else: return self.storage.retrieve_with_metadata(norm_key)
def _miss(self, key, expiration): if self.verbose: print(styled("ResourceLoader: Generating", "white", "red", "bold"), end=' ') print(key) value = self.load(key) if self.enabled: self.set_value(key, value, expiration) return value
def get_access_token_from_refresh_token(self, refresh_token): """ Gets a new access token from a refresh token. :return: A dict respresenting an access token. Its 'access_token' key is a valid access token """ TOKEN_URL = "https://accounts.google.com/o/oauth2/token" params = { "client_id": self.client_id, "client_secret": self.client_secret, "refresh_token": refresh_token, "grant_type": "refresh_token" } request = urllib.request.Request(url=TOKEN_URL, data=urllib.parse.urlencode(params)) try: json_file = urllib.request.urlopen(request).read() except urllib.error.HTTPError as e: if self.debug_mode: print( styled( "Error while obtaining a Google access token from a " "refresh token:", "magenta")) print(e.read()) raise access_token = json.loads(json_file) if self.debug_mode: print(styled("Google access token from refresh token:", "magenta"), end=' ') print(access_token) return access_token
def _apply_changes(self, item): # Remove those instances that have been dettached from an integral # reference @when(item.changed) def delete_replaced_integral_children(event): if isinstance(event.member, Reference) \ and event.member.integral \ and event.previous_value is not None: delete_validating(event.previous_value) try: stack_node = self.stack_node from cocktail.styled import styled print styled(stack_node.form_data, "pink") stack_node.import_form_data(stack_node.form_data, item) print styled(stack_node.form_data, "bright_green") item.insert() stack_node.saving( user = get_current_user(), changeset = ChangeSet.current ) finally: item.changed.remove(delete_replaced_integral_children)
def create_links(file, links = None, encoding = None): if not file.is_inserted: return anonymous = User.require_instance(qname = "woost.anonymous_user") if not file.is_accessible(anonymous): return if links is None: links = get_links(file) linked_file = file.file_path linked_file = encode_filename(linked_file, encoding) for link in links: link = encode_filename(link, encoding) if debug: print styled("STATIC PUBLICATION", "white", "green"), print "Adding link for #%s (%s -> %s)" % ( styled(file.id, style = "bold"), styled(link, "yellow"), styled(linked_file, "brown") ) # Delete the link if it already existed try: os.remove(link) except OSError: pass # Create the new link os.symlink(linked_file, link)
def step2(self): credentials = session.get(SESSION_PREFIX + "credentials") if not credentials or credentials.access_token_expired: redirect(self.step_url(1)) http_auth = credentials.authorize(httplib2.Http()) oauth2_service = discovery.build('oauth2', 'v2', http_auth) user_data = oauth2_service.userinfo().get().execute() if self.provider.debug_mode: print(styled("Google user profile:", "magenta"), user_data) self.provider.login(user_data) del session[SESSION_PREFIX + "credentials"] redirect(self.target_url)