def _remove_dir_contents(path, pattern=None):
# os.path.lexists is not supported on windows
from cocktail.styled import styled
exists = getattr(os.path, "lexists", os.path.exists)
if exists(path):
if pattern:
items = glob(os.path.join(path, pattern))
else:
items = os.listdir(path)
for item in items:
item_path = os.path.join(path, item)
if debug:
print styled(" " * 4 + item_path, "red")
try:
if os.path.isdir(item_path):
rmtree(item_path)
else:
os.remove(item_path)
except OSError, ex:
if debug:
print styled(ex, "red")
def set_expiration(self, key: CacheKey, expiration: Optional[int]):
"""Sets the expiration assigned to the given key.
:param key: The key to update the expiration for.
:param expiration: The new expiration date for the key. Can be set to
None to disable time based expiration for the key; otherwise, check
the `~cocktail.caching.utils.normalize_expiration` function for all
the possible ways of specifying the expiration date for the key.
:raises cocktail.caching.CacheKeyError: Raised if the key is not
contained in the storage or has already expired.
:raises ValueError: Raised if the given expiration key is not valid
(ie. is not an integer timestamp, or is set in the past).
"""
if not self.enabled or self.storage is None:
raise CacheKeyError(key)
norm_key = self.normalize_key(key)
if self.verbose:
print((styled("CACHE", "white", "dark_gray") + " " +
styled("Set expiration", "pink", style="bold") + "\n" +
styled(" Key:", "light_gray", style="bold") + " " +
norm_key + "\n" +
styled(" Expiration:", "light_gray", style="bold") + " " +
str(expiration) + "\n"))
if expiration is not None:
expiration = normalize_expiration(expiration)
self.storage.set_expiration(norm_key, expiration)
def clear(self, scope: Scope = whole_cache):
"""Discards all keys in the cache matching the given scope.
:param scope: A selector indicating the keys that should be removed. It
can take the following forms:
- `~cocktail.caching.whole_cache` will remove all the entries
in the cache.
- A string will be treated as a single tag, and will delete all
entries tagged with said tag.
- A tuple of strings will be interpreted as an intersection of
tags: any entry tagged with *all* of the tags in the tuple
will be deleted.
- Collections will be treated as a union of other selectors:
all entries matching *any* of the selectors in the collection
will be removed.
:raises TypeError: Raised if the ``scope`` parameter is given a value
of a wrong type.
"""
if self.enabled and self.storage is not None:
if self.verbose:
print(
(styled("CACHE", "white", "dark_gray") + " " +
styled("Clear", "magenta", style="bold") + "\n" +
styled(" Scope:", "light_gray", style="bold") + " " +
("whole cache" if scope is whole_cache else repr(scope)) +
"\n"))
self.storage.clear(scope=normalize_scope(scope))
def discard(self, key: CacheKey) -> bool:
"""Removes the given key from the cache, if it exists.
This method is similar to `.remove`, but it will return a boolean
indicating wether the removal succeeded (`.remove` raises an exception
if it can't find the requested key in the cache).
:param key: The key to remove.
:return: True if the key was removed from the cache, False if it
wasn't contained on the cache, or if it had already expired.
"""
if not self.enabled or self.storage is None:
return False
norm_key = self.normalize_key(key)
if self.storage.discard(norm_key):
if self.verbose:
print((styled("CACHE", "white", "dark_gray") + " " +
styled("Remove", "magenta", style="bold") + "\n" +
styled(" Key:", "light_gray", style="bold") + " " +
norm_key + "\n"))
return True
else:
return False
def print_memory_usage(self):
if self.memory_limit:
usage = self.memory_usage
limit = self.memory_limit
ratio = usage / limit
percent = int(ratio * 100)
bar = (
"%s / %s" % (
format_bytes(usage),
format_bytes(limit)
)
).rjust(self.memory_usage_bar_width)
for threshold, bar_color in self.memory_usage_colors:
if percent >= threshold:
break
bar_width = int(ratio * self.memory_usage_bar_width)
info = (
styled(bar[:bar_width], "white", bar_color)
+ styled(bar[bar_width:], "white", "dark_gray")
)
else:
info = format_bytes(self.memory_usage)
print("Memory usage: %s\n" % info)
def _is_current(self, resource, invalidation=None, verbose=False):
# Expiration
if resource.has_expired(default_expiration=self.expiration):
if verbose:
print(styled("ResourceLoader: Resource expired", "white",
"brown", "bold"),
resource.key,
end=' ')
if resource.expiration is None:
print(self.expiration)
else:
print(resource.expiration)
return False
# Invalidation
if callable(invalidation):
invalidation = invalidation()
if invalidation is not None:
if isinstance(invalidation, datetime):
invalidation = mktime(invalidation.timetuple())
if invalidation > resource.creation:
if verbose:
print(styled("ResourceLoader: Resource invalidated",
"white", "brown", "bold"),
end=' ')
print(resource.key, invalidation)
return False
return True
def step1(self, code=None, target_url=None, **kwargs):
if target_url:
self.target_url = target_url
session[SESSION_PREFIX + "target_url"] = target_url
flow = OAuth2WebServerFlow(self.provider.client_id,
self.provider.client_secret,
self.provider.scope,
redirect_uri=self.step_url(1))
flow.params["access_type"] = self.provider.access_type
if not code:
redirect(flow.step1_get_authorize_url())
if self.provider.debug_mode:
print(styled("Google authorization code:", "magenta"), code)
credentials = flow.step2_exchange(code)
session[SESSION_PREFIX + "credentials"] = credentials
if self.provider.debug_mode:
print(styled("Google refresh token:", "magenta"), end=' ')
print(credentials.refresh_token)
print(styled("Google access token:", "magenta"), end=' ')
print(credentials.access_token)
redirect(self.step_url(2))
def print_tree(self, indent=""):
print(indent + styled(repr(self), "slate_blue"))
indent += " "
for group in self.groups:
group.print_tree(indent)
for item in self.items:
print(indent + styled(item, "pink"))
def extract(self, member, value, language=None):
if value is None:
return False
if self.verbose:
print((" " * 4 * len(self.__stack)) + "Extracting", end=' ')
print(styled(member, style="bold"), end=' ')
print("from", end=' ')
print(styled(value, style="bold"), end=' ')
if language:
print("in language", styled(language, style="bold"))
else:
print()
is_object = isinstance(type(value), Schema)
if is_object:
if value in self.__visited:
return False
else:
self.__visited.add(value)
node = TextExtractorNode(member, language, value, [],
self.__stack[-1] if self.__stack else None)
self.__stack.append(node)
try:
member.extract_searchable_text(self)
finally:
self.__stack.pop(-1)
return True
def _insert_into_manifest(e):
obj = e.source
if obj.synchronizable:
global_id = e.source.global_id
if global_id:
get_manifest()[global_id] = None
if debug:
print styled("Manifest: declared " + obj.global_id,
"bright_green")
def show_blame(self):
for cls, tb_count in self.blame.iteritems():
print styled(cls, "slate_blue")
for tb, count in tb_count.most_common():
print
print "%s reads" % styled(count, style="bold")
for fname, line, func, code in tb:
print " %s %s:%d" % (func, fname, line)
print " " + styled(code, "light_gray")
def _invalidate_manifest(e):
from cocktail.styled import styled
obj = e.source
if not obj.is_inserted:
return
# Elegibility for synchronization changed
if e.member is Item.synchronizable:
# Object becomes synchronizable: add it to the manifest
if e.value:
if obj.global_id:
get_manifest()[obj.global_id] = None
if debug:
print styled("Manifest: declared " + obj.global_id,
"bright_green")
# Object ceases to be synchronizable: remove it from the manifest
else:
get_manifest().pop(obj.global_id, None)
if debug:
print styled("Manifest: removed " + obj.global_id, "magenta")
return
elif not obj.synchronizable:
return
# Object global identifier changed
if e.member is Item.global_id:
manifest = get_manifest()
if e.previous_value:
manifest.pop(e.previous_value, None)
if debug:
print styled("Manifest: removed " + obj.global_id, "magenta")
if e.value:
manifest[e.value] = None
if debug:
print styled("Manifest: declared " + obj.global_id,
"bright_green")
return
elif not obj.global_id:
return
# Changing a synchronizable member: invalidate the stored object's hash in
# the manifest, if any
if e.member.synchronizable:
get_manifest()[obj.global_id] = None
if debug:
print styled("Manifest: invalidated " + obj.global_id, "cyan")
def set_value(self, key, value, expiration=None):
if self.verbose:
print(styled("ResourceLoader: Storing", "white", "pink", "bold"),
end=' ')
print(key, end=' ')
if expiration is None:
print()
else:
print(styled("expiration:", "pink"), expiration)
self.__resources[key] = Resource(key, value, expiration)
def list_action(self):
tasks = list(self.tasks)
tasks.sort()
tasks = defaultdict(
(lambda: defaultdict(
(lambda: defaultdict(list))
))
)
for action, publishable, language in self.tasks:
tasks[action][publishable.__class__][publishable].append(language)
for action in sorted(tasks):
print()
print(
styled(
action.upper().ljust(120),
"white",
"green" if action == "post" else "red",
"bold"
)
)
for cls, cls_tasks in tasks[action].iteritems():
cls_label = translations(cls)
print()
print(
styled(
cls_label.ljust(120),
"white",
"dark_gray",
"bold"
)
)
for publishable, languages in cls_tasks.iteritems():
print(str(publishable.id).ljust(10), end=" ")
print(
styled(
translations(
publishable,
discard_generic_translation=True,
language=languages[0]
)[:36].ljust(38),
"slate_blue"
),
end=" "
)
if languages != [None]:
print(styled(", ".join(languages), "pink"))
else:
print()
def remove_links(file, links = None, encoding = None):
if links is None:
links = get_links(file)
for link in links:
link = encode_filename(link, encoding)
if os.path.lexists(link):
if debug:
print styled("STATIC PUBLICATION", "white", "red"),
print "Removing link for #%s (%s)" % (
styled(file.id, style = "bold"),
styled(link, "yellow")
)
os.remove(link)
def get_refresh_token(self, code, redirect_uri=None):
"""
Gets an authorization token with access_token and refresh_token from
an authorization code.
:param code: A valid authorization code with access_type offline
:return: A dict representing the authorization token. The
'access_token' key is a valid access token and the 'refresh_token'
key can be used to generate new valid access tokens
"""
TOKEN_URL = "https://accounts.google.com/o/oauth2/token"
if not redirect_uri:
website = (app.website or Configuration.instance.websites[0])
redirect_uri = "http://%s/google_oauth/%d/refresh_token" % (
website.hosts[0], self.id)
params = {
"code": code,
"client_id": self.client_id,
"client_secret": self.client_secret,
"redirect_uri": redirect_uri,
"grant_type": "authorization_code"
}
request = urllib.request.Request(url=TOKEN_URL,
data=urllib.parse.urlencode(params))
json_file = urllib.request.urlopen(request).read()
auth_token = json.loads(json_file)
if self.debug_mode:
print(styled("Google refresh token:", "magenta"), auth_token)
return auth_token
def transaction(action,
action_args=(),
action_kwargs=None,
max_attempts=3,
before_retrying=None,
desist=None):
if action_kwargs is None:
action_kwargs = {}
for i in range(max_attempts):
if i > 0:
if verbose:
print(
styled(
"Retrying transaction %s (%d/%d)" %
(action, i, max_attempts - 1), {
1: "yellow",
2: "brown",
3: "red"
}.get(i, "violet")))
if before_retrying is not None:
before_retrying(*action_args, **action_kwargs)
if desist is not None and desist(*action_args, **action_kwargs):
return desisted
try:
rvalue = action(*action_args, **action_kwargs)
datastore.commit()
return rvalue
except ConflictError:
datastore.sync() # implicit abort
raise
def store(self,
key: CacheKey,
value: Any,
expiration: Optional[int] = None,
tags: Optional[Set[str]] = None):
"""Inserts or updates a value in the storage.
If the key already existed, the given value, expiration and tags will
be used to replace the old ones.
:param key: A string that will uniquely identify the value in the
cache, making it possible to `.retrieve` it later.
:param value: The value to associate to this key.
:param expiration: An optional parameter that limits the maximum
life span of the key in the storage. See the
`~cocktail.caching.utils.normalize_expiration` function for a
detailed explanation of all the forms the parameter can take.
:param tags: An optional set of tags to attach to this key. If given,
it should be expressed as a collection of strings, each one
representing a single tag.
"""
if not self.enabled or self.storage is None:
return
norm_key = self.normalize_key(key)
if self.verbose:
lines = [
styled("CACHE", "white", "dark_gray") + " " +
styled("Storing", "slate_blue", style="bold"),
styled(" Key:", "light_gray", style="bold") + " " + norm_key
]
if expiration is not None:
lines.append(
styled(" Expiration:", "light_gray", style="bold") + " " +
str(expiration))
if tags is not None:
lines.append(
styled(" Tags:", "light_gray", style="bold") + " " +
str(sorted(tags)))
lines.append("")
print("\n".join(lines))
if expiration is not None:
expiration = normalize_expiration(expiration)
self.storage.store(norm_key, value, expiration=expiration, tags=tags)
def clear_image_cache(item=None, factory=None):
if not app.root:
return
if debug:
from cocktail.styled import styled
print styled("Clearing image cache", "red"),
if item:
print styled("Item:", "light_gray"),
print styled(item, "red", style="bold"),
if factory:
print styled("Factory:", "light_gray"),
print styled(factory, "red", style="bold"),
print
# Remove the full cache
if item is None and factory is None:
_remove_dir_contents(app.path("image-cache"))
_remove_dir_contents(app.path("static", "images"))
# Selective drop: per item and/or factory
else:
paths = []
if item is not None:
paths.append(app.path("image-cache", str(item.id)))
paths.append(app.path("static", "images", str(item.id)))
else:
for base in (app.path("image-cache"), app.path("static",
"images")):
for item in os.listdir(base):
path = os.path.join(base, item)
if os.path.isdir(path):
paths.append(path)
if factory is None:
pattern = None
else:
pattern = factory + ".*"
for path in paths:
_remove_dir_contents(path, pattern)
def step1(self, code=None, target_url=None, **kwargs):
self.check_step1_errors(**kwargs)
if target_url:
self.target_url = target_url
session[SESSION_PREFIX + "target_url"] = target_url
if not code:
params = {
'client_id': self.provider.client_id,
'redirect_uri': self.step_url(1),
'scope': ','.join(self.provider.scope)
}
login_uri = 'https://www.facebook.com/dialog/oauth?' + \
urllib.parse.urlencode(params)
redirect(login_uri)
if self.provider.debug_mode:
print(styled("Facebook authorization code:", "magenta"), code)
params = {
'client_id': self.provider.client_id,
'redirect_uri': self.step_url(1),
'client_secret': self.provider.client_secret,
'code': code
}
token_uri = 'https://graph.facebook.com/v2.3/oauth/access_token?' \
+ urllib.parse.urlencode(params)
json_file = urllib.request.urlopen(token_uri).readline()
token_data = json.loads(json_file)
if not token_data.get("access_token"):
raise FacebookOAuthBadResponseException(
token_data, "Expected an 'access_token' key")
session[SESSION_PREFIX + "credentials"] = token_data
if self.provider.debug_mode:
print(styled("Facebook token data:", "magenta"), end=' ')
print(token_data)
redirect(self.step_url(2))
def drop_weight(self) -> Optional[CacheKey]:
"""Removes an entry from the cache, in order to free resources.
:return: The key that has been removed, or ``None`` if the cache is
empty.
"""
if not self.enabled or self.storage is None:
return
key = self.storage.drop_weight()
if key and self.verbose:
print((styled("CACHE", "white", "dark_gray") + " " +
styled("Drop weight", "magenta", style="bold") + "\n" +
styled(" Key:", "light_gray", style="bold") + " " + key +
"\n"))
return key
def clear_after_commit(self, scope: Scope = whole_cache):
"""Schedules a cache invalidation operation after the current ZODB
transaction is committed.
This method can be called multiple times per transaction, with each
successive call widening the scope that will be affected once the
transaction is completed. That said, if the scope is set to
`~cocktail.caching.whole_cache`, the scope indicated by past or future
invocations will be disregarded, and the whole cache will be cleared
once the transaction is completed.
:param scope: Determines the keys that will be removed once the
transaction is completed. See the `.clear` method for details on
its accepted formats.
"""
if not self.enabled or self.storage is None or not scope:
return
key = (TRANSACTION_KEY, id(self))
from cocktail.persistence import datastore
transaction_invalidation_scope = datastore.get_transaction_value(key)
if transaction_invalidation_scope is whole_cache:
return
elif transaction_invalidation_scope is None:
datastore.unique_after_commit_hook(
TRANSACTION_KEY, _cache_invalidation_commit_handler, self)
scope = normalize_scope(scope)
if transaction_invalidation_scope is None:
datastore.set_transaction_value(key, scope)
elif scope is whole_cache:
datastore.set_transaction_value(key, scope)
else:
transaction_invalidation_scope.update(scope)
if self.verbose:
print(
(styled("CACHE", "white", "dark_gray") + " " +
styled("Clear after commit", "magenta", style="bold") + "\n" +
styled(" Scope:", "light_gray", style="bold") + " " +
("whole cache" if scope is whole_cache else repr(scope)) +
"\n"))
def main():
params = argv[1:]
if not params:
print "Usage: %s filelist" % argv[0]
exit(1)
for param in params:
for path in glob(param):
print "Importing " + styled(path, style="bold") + "...",
try:
file = File.from_path(
path, resource_filename("_PROJECT_NAME_", "upload"))
file.insert()
except Exception, ex:
print styled(str(ex), "red")
else:
print styled("File #" + str(file.id), "green")
def clear_audio_cache(item=None, encoder_id=None):
if debug:
from cocktail.styled import styled
print styled("Clearing audio cache", "red"),
if item:
print styled("Item:", "light_gray"),
print styled(item, "red", style="bold"),
if encoder_id:
print styled("Encoder:", "light_gray"),
print styled(encoder_id, "red", style="bold"),
print
# Remove the full cache
if item is None and encoder_id is None:
_remove_dir_contents(app.path("audio-cache"))
_remove_dir_contents(app.path("static", "audio"))
# Selective drop: per item and/or encoder
else:
paths = []
if item is not None:
paths.append(app.path("audio-cache", str(item.id)))
paths.append(app.path("static", "audio", str(item.id)))
else:
for base in (app.path("audio-cache"), app.path("static", "audio")):
for item in os.listdir(base):
path = os.path.join(base, item)
if os.path.isdir(path):
paths.append(path)
if encoder_id is None:
pattern = None
else:
pattern = encoder_id + ".*"
for path in paths:
_remove_dir_contents(path, pattern)
def retrieve_with_metadata(self,
key: CacheKey) -> Tuple[Any, int, Set[str]]:
"""Obtains the value, expiration and tags for the given key.
:param key: The key to retrieve.
:return: A tuple containing the value, expiration and tags associated
with the indicated key.
:raises cocktail.caching.CacheKeyError: Raised if the key is not
present in the `.storage` for the cache, or if it has expired.
"""
if not self.enabled or self.storage is None:
raise CacheKeyError(key)
norm_key = self.normalize_key(key)
if self.verbose:
try:
value = self.storage.retrieve_with_metadata(norm_key)
except CacheKeyError:
print((styled("CACHE", "white", "dark_gray") + " " +
styled("Miss", "red", style="bold") + "\n" +
styled(" Key:", "light_gray", style="bold") + " " +
norm_key + "\n"))
raise
else:
print((styled("CACHE", "white", "dark_gray") + " " +
styled("Hit", "bright_green", style="bold") + "\n" +
styled(" Key:", "light_gray", style="bold") + " " +
norm_key + "\n"))
return value
else:
return self.storage.retrieve_with_metadata(norm_key)
def _miss(self, key, expiration):
if self.verbose:
print(styled("ResourceLoader: Generating", "white", "red", "bold"),
end=' ')
print(key)
value = self.load(key)
if self.enabled:
self.set_value(key, value, expiration)
return value
def get_access_token_from_refresh_token(self, refresh_token):
"""
Gets a new access token from a refresh token.
:return: A dict respresenting an access token. Its 'access_token' key
is a valid access token
"""
TOKEN_URL = "https://accounts.google.com/o/oauth2/token"
params = {
"client_id": self.client_id,
"client_secret": self.client_secret,
"refresh_token": refresh_token,
"grant_type": "refresh_token"
}
request = urllib.request.Request(url=TOKEN_URL,
data=urllib.parse.urlencode(params))
try:
json_file = urllib.request.urlopen(request).read()
except urllib.error.HTTPError as e:
if self.debug_mode:
print(
styled(
"Error while obtaining a Google access token from a "
"refresh token:", "magenta"))
print(e.read())
raise
access_token = json.loads(json_file)
if self.debug_mode:
print(styled("Google access token from refresh token:", "magenta"),
end=' ')
print(access_token)
return access_token
def _apply_changes(self, item):
# Remove those instances that have been dettached from an integral
# reference
@when(item.changed)
def delete_replaced_integral_children(event):
if isinstance(event.member, Reference) \
and event.member.integral \
and event.previous_value is not None:
delete_validating(event.previous_value)
try:
stack_node = self.stack_node
from cocktail.styled import styled
print styled(stack_node.form_data, "pink")
stack_node.import_form_data(stack_node.form_data, item)
print styled(stack_node.form_data, "bright_green")
item.insert()
stack_node.saving(
user = get_current_user(),
changeset = ChangeSet.current
)
finally:
item.changed.remove(delete_replaced_integral_children)
def create_links(file, links = None, encoding = None):
if not file.is_inserted:
return
anonymous = User.require_instance(qname = "woost.anonymous_user")
if not file.is_accessible(anonymous):
return
if links is None:
links = get_links(file)
linked_file = file.file_path
linked_file = encode_filename(linked_file, encoding)
for link in links:
link = encode_filename(link, encoding)
if debug:
print styled("STATIC PUBLICATION", "white", "green"),
print "Adding link for #%s (%s -> %s)" % (
styled(file.id, style = "bold"),
styled(link, "yellow"),
styled(linked_file, "brown")
)
# Delete the link if it already existed
try:
os.remove(link)
except OSError:
pass
# Create the new link
os.symlink(linked_file, link)
def step2(self):
credentials = session.get(SESSION_PREFIX + "credentials")
if not credentials or credentials.access_token_expired:
redirect(self.step_url(1))
http_auth = credentials.authorize(httplib2.Http())
oauth2_service = discovery.build('oauth2', 'v2', http_auth)
user_data = oauth2_service.userinfo().get().execute()
if self.provider.debug_mode:
print(styled("Google user profile:", "magenta"), user_data)
self.provider.login(user_data)
del session[SESSION_PREFIX + "credentials"]
redirect(self.target_url)
