def __init__(self, adjacency_map):
"""
TopologicalSorter constructor.
@param adjacency_map: dict form adjacency map
@type adjacency_map: dict
"""
object.__init__(self)
self.__adjacency_map = adjacency_map
self.__stack = Lifo()
def __init__(self, uri):
"""
EntropyTransceiver constructor, just pass the friggin URI(tm).
@param uri: URI to handle
@type uri: string
"""
self._uri = uri
self._speed_limit = 0
self._verbose = False
self._timeout = None
self._silent = None
self._output_interface = None
self.__with_stack = Lifo()
def __topological_sort(self, graph):
"""
Effectively executes topological sorting on given graph.
"""
# initialize count map
count = dict((node, 0) for node in graph)
for node in graph:
for successor in graph[node]:
count[successor] += 1
ready_stack = Lifo()
for node in graph:
if count[node] == 0:
ready_stack.push(node)
dep_level = 1
result = {}
while ready_stack.is_filled():
node = ready_stack.pop()
result[dep_level] = node
dep_level += 1
for successor in graph[node]:
count[successor] -= 1
if count[successor] == 0:
ready_stack.push(successor)
return result
def init_singleton(self):
"""
Singleton overloaded method. Equals to __init__.
This is the place where all the properties initialization
takes place.
"""
self.__copy = copy
self.__alive = False
self.__cache_writer = None
self.__cache_buffer = Lifo()
self.__stashing_cache = {}
self.__inside_with_stmt = 0
self.__dump_data_lock = threading.Lock()
self.__worker_sem = threading.Semaphore(0)
# this lock ensures that all the writes are hold while it's acquired
self.__enter_context_lock = threading.RLock()
def __topological_sort(self, graph):
"""
Effectively executes topological sorting on given graph.
"""
# initialize count map
count = dict((node, 0) for node in graph)
for node in graph:
for successor in graph[node]:
count[successor] += 1
ready_stack = Lifo()
for node in graph:
if count[node] == 0:
ready_stack.push(node)
dep_level = 1
result = {}
while ready_stack.is_filled():
node = ready_stack.pop()
result[dep_level] = node
dep_level += 1
for successor in graph[node]:
count[successor] -= 1
if count[successor] == 0:
ready_stack.push(successor)
return result
def __init__(self, adjacency_map):
"""
TopologicalSorter constructor.
@param adjacency_map: dict form adjacency map
@type adjacency_map: dict
"""
object.__init__(self)
self.__adjacency_map = adjacency_map
self.__stack = Lifo()
def init_singleton(self):
"""
Singleton overloaded method. Equals to __init__.
This is the place where all the properties initialization
takes place.
"""
self.__copy = copy
self.__alive = False
self.__cache_writer = None
self.__cache_buffer = Lifo()
self.__stashing_cache = {}
self.__inside_with_stmt = 0
self.__dump_data_lock = threading.Lock()
self.__worker_sem = threading.Semaphore(0)
# this lock ensures that all the writes are hold while it's acquired
self.__enter_context_lock = threading.RLock()
def _graph_to_stdout(entropy_client, graph, start_item,
item_translation_callback,
show_already_pulled_in, quiet):
if not quiet:
entropy_client.output("="*40, level="generic")
sorted_data = graph.solve_nodes()
stack = Lifo()
for dep_level in sorted(sorted_data.keys(), reverse = True):
stack.push(sorted_data[dep_level])
# required to make sure that our first pkg is user required one
stack.push((start_item,))
out_data = {
'cache': set(),
'lvl': 0,
'txc_cb': item_translation_callback,
'show_already_pulled_in': show_already_pulled_in,
}
first_tree_item = True
while stack.is_filled():
stack_items = stack.pop()
# cleanup already printed items
items = [x for x in stack_items if x not in out_data['cache']]
if not items:
continue
out_data['cache'].update(stack_items)
# print items and its deps
for item in items:
old_level = out_data['lvl']
_print_graph_item_deps(
entropy_client, item, out_data, colorize = blue)
out_data['lvl'] = old_level
if first_tree_item:
out_data['lvl'] += 1
first_tree_item = False
del stack
def _graph_to_stdout(entropy_client, graph, start_item,
item_translation_callback,
show_already_pulled_in, quiet):
if not quiet:
entropy_client.output("="*40, level="generic")
sorted_data = graph.solve_nodes()
stack = Lifo()
for dep_level in sorted(sorted_data.keys(), reverse = True):
stack.push(sorted_data[dep_level])
# required to make sure that our first pkg is user required one
stack.push((start_item,))
out_data = {
'cache': set(),
'lvl': 0,
'txc_cb': item_translation_callback,
'show_already_pulled_in': show_already_pulled_in,
}
first_tree_item = True
while stack.is_filled():
stack_items = stack.pop()
# cleanup already printed items
items = [x for x in stack_items if x not in out_data['cache']]
if not items:
continue
out_data['cache'].update(stack_items)
# print items and its deps
for item in items:
old_level = out_data['lvl']
_print_graph_item_deps(
entropy_client, item, out_data, colorize = blue)
out_data['lvl'] = old_level
if first_tree_item:
out_data['lvl'] += 1
first_tree_item = False
del stack
def _get_flat_deps(self, ids_to_check, get_deps_func):
"""
Return a set (frozenset) of package IDs that are dependencies
of provided packages, recursively.
"""
stack = Lifo()
result_deps = set()
for pkg_id in ids_to_check:
stack.push(pkg_id)
while stack.is_filled():
pkg_id = stack.pop()
if pkg_id in result_deps:
continue
result_deps.add(pkg_id)
for dep_id in get_deps_func(pkg_id):
if dep_id not in result_deps:
stack.push(dep_id)
return frozenset(result_deps)
def _get_flat_deps(self, ids_to_check, get_deps_func):
"""
Return a set (frozenset) of package IDs that are dependencies
of provided packages, recursively.
"""
stack = Lifo()
result_deps = set()
for pkg_id in ids_to_check:
stack.push(pkg_id)
while stack.is_filled():
pkg_id = stack.pop()
if pkg_id in result_deps:
continue
result_deps.add(pkg_id)
for dep_id in get_deps_func(pkg_id):
if dep_id not in result_deps:
stack.push(dep_id)
return frozenset(result_deps)
class EntropyCacher(Singleton):
# Max number of cache objects written at once
_OBJS_WRITTEN_AT_ONCE = 250
# Number of seconds between cache writeback to disk
WRITEBACK_TIMEOUT = 5
# If True, in-ram cache will be used to mitigate
# concurrent push/pop executions with push() not
# yet able to write data to disk.
STASHING_CACHE = True
"""
Entropy asynchronous and synchronous cache writer
and reader. This class is a Singleton and contains
a thread doing the cache writes asynchronously, thus
it must be stopped before your application is terminated
calling the stop() method.
Sample code:
>>> # import module
>>> from entropy.cache import EntropyCacher
...
>>> # first EntropyCacher load, start it
>>> cacher = EntropyCacher()
>>> cacher.start()
...
>>> # now store something into its cache
>>> cacher.push('my_identifier1', [1, 2, 3])
>>> # now store something synchronously
>>> cacher.push('my_identifier2', [1, 2, 3], async_mode = False)
...
>>> # now flush all the caches to disk, and make sure all
>>> # is written
>>> cacher.sync()
...
>>> # now fetch something from the cache
>>> data = cacher.pop('my_identifier1')
[1, 2, 3]
...
>>> # now discard all the cached (async_mode) writes
>>> cacher.discard()
...
>>> # and stop EntropyCacher
>>> cacher.stop()
"""
class SemaphoreTimeScheduled(TimeScheduled):
def __init__(self, sem, *args, **kwargs):
self._sem = sem
TimeScheduled.__init__(self, *args, **kwargs)
def kill(self):
self._sem.release()
return TimeScheduled.kill(self)
def init_singleton(self):
"""
Singleton overloaded method. Equals to __init__.
This is the place where all the properties initialization
takes place.
"""
self.__copy = copy
self.__alive = False
self.__cache_writer = None
self.__cache_buffer = Lifo()
self.__stashing_cache = {}
self.__inside_with_stmt = 0
self.__dump_data_lock = threading.Lock()
self.__worker_sem = threading.Semaphore(0)
# this lock ensures that all the writes are hold while it's acquired
self.__enter_context_lock = threading.RLock()
def __enter__(self):
"""
When used with the with statement, pause cacher on-disk writes.
"""
self.__enter_context_lock.acquire()
self.__inside_with_stmt += 1
def __exit__(self, exc_type, exc_value, traceback):
"""
When used with the with statement, pause cacher on-disk writes.
"""
self.__inside_with_stmt -= 1
self.__enter_context_lock.release()
def __copy_obj(self, obj):
"""
Return a copy of an object done by the standard
library "copy" module.
@param obj: object to copy
@type obj: any Python object
@rtype: copied object
@return: copied object
"""
return self.__copy.deepcopy(obj)
def __cacher(self, run_until_empty=False, sync=False, _loop=False):
"""
This is where the actual asynchronous copy takes
place. __cacher runs on a different threads and
all the operations done by this are atomic and
thread-safe. It just loops over and over until
__alive becomes False.
"""
try:
if self.__inside_with_stmt != 0:
return
except AttributeError:
# interpreter shutdown
pass
# make sure our set delay is respected
try:
self.__cache_writer.set_delay(EntropyCacher.WRITEBACK_TIMEOUT)
except AttributeError:
# can be None
pass
# sleep if there's nothing to do
if _loop:
try:
# CANBLOCK
self.__worker_sem.acquire()
# we just consumed one acquire()
# that was dedicated to actual data,
# put it back
self.__worker_sem.release()
except AttributeError:
pass
def _commit_data(_massive_data):
for (key, cache_dir), data in _massive_data:
d_o = entropy.dump.dumpobj
if d_o is not None:
d_o(key, data, dump_dir=cache_dir)
while self.__alive or run_until_empty:
if const_debug_enabled():
const_debug_write(
__name__,
"EntropyCacher.__cacher: loop: %s, alive: %s, empty: %s" %
(
_loop,
self.__alive,
run_until_empty,
))
with self.__enter_context_lock:
massive_data = []
try:
massive_data_count = EntropyCacher._OBJS_WRITTEN_AT_ONCE
except AttributeError: # interpreter shutdown
break
while massive_data_count > 0:
if _loop:
# extracted an item from worker_sem
# call down() on the semaphore without caring
# can't sleep here because we're in a critical region
# holding __enter_context_lock
self.__worker_sem.acquire(False)
massive_data_count -= 1
try:
data = self.__cache_buffer.pop()
except (
ValueError,
TypeError,
):
# TypeError is when objects are being destroyed
break # stack empty
massive_data.append(data)
if not massive_data:
break
task = ParallelTask(_commit_data, massive_data)
task.name = "EntropyCacherCommitter"
task.daemon = not sync
task.start()
if sync:
task.join()
if const_debug_enabled():
const_debug_write(
__name__,
"EntropyCacher.__cacher [%s], writing %s objs" % (
task,
len(massive_data),
))
if EntropyCacher.STASHING_CACHE:
for (key, cache_dir), data in massive_data:
try:
del self.__stashing_cache[(key, cache_dir)]
except (
AttributeError,
KeyError,
):
continue
del massive_data[:]
del massive_data
@classmethod
def current_directory(cls):
"""
Return the path to current EntropyCacher cache storage directory.
"""
return entropy.dump.D_DIR
def start(self):
"""
This is the method used to start the asynchronous cache
writer but also the whole cacher. If this method is not
called, the instance will always trash and cache write
request.
@return: None
"""
self.__cache_buffer.clear()
self.__cache_writer = EntropyCacher.SemaphoreTimeScheduled(
self.__worker_sem,
EntropyCacher.WRITEBACK_TIMEOUT,
self.__cacher,
_loop=True)
self.__cache_writer.daemon = True
self.__cache_writer.name = "EntropyCacheWriter"
self.__cache_writer.set_delay_before(True)
self.__cache_writer.start()
while not self.__cache_writer.isAlive():
continue
self.__alive = True
def is_started(self):
"""
Return whether start is called or not. This equals to
checking if the cacher is running, thus is writing cache
to disk.
@return: None
"""
return self.__alive
def stop(self):
"""
This method stops the execution of the cacher, which won't
accept cache writes anymore. The thread responsible of writing
to disk is stopped here and the Cacher will be back to being
inactive. A watchdog will avoid the thread to freeze the
call if the write buffer is overloaded.
@return: None
"""
self.__alive = False
if self.__cache_writer is not None:
self.__cache_writer.kill()
# make sure it unblocks
self.__worker_sem.release()
self.__cache_writer.join()
self.__cache_writer = None
self.sync()
def sync(self):
"""
This method can be called anytime and forces the instance
to flush all the cache writes queued to disk. If wait == False
a watchdog prevents this call to get stuck in case of write
buffer overloads.
"""
self.__cacher(run_until_empty=True, sync=True)
def discard(self):
"""
This method makes buffered cache to be discarded synchronously.
@return: None
"""
self.__cache_buffer.clear()
self.__stashing_cache.clear()
def save(self, key, data, cache_dir=None):
"""
Save data object to cache asynchronously and in any case.
This method guarantees that cached data is stored even if cacher
is not started. If data cannot be stored, IOError will be raised.
@param key: cache data identifier
@type key: string
@param data: picklable object
@type data: any picklable object
@keyword cache_dir: alternative cache directory
@type cache_dir: string
"""
if cache_dir is None:
cache_dir = self.current_directory()
try:
with self.__dump_data_lock:
entropy.dump.dumpobj(key,
data,
dump_dir=cache_dir,
ignore_exceptions=False)
except (EOFError, IOError, OSError) as err:
raise IOError("cannot store %s to %s. err: %s" %
(key, cache_dir, repr(err)))
def push(self, key, data, async_mode=True, cache_dir=None):
"""
This is the place where data is either added
to the write queue or written to disk (if async_mode == False)
only and only if start() method has been called.
@param key: cache data identifier
@type key: string
@param data: picklable object
@type data: any picklable object
@keyword async_mode: store cache asynchronously or not
@type async_mode: bool
@keyword cache_dir: alternative cache directory
@type cache_dir: string
"""
if not self.__alive:
return
if cache_dir is None:
cache_dir = self.current_directory()
if async_mode:
try:
obj_copy = self.__copy_obj(data)
self.__cache_buffer.push((
(
key,
cache_dir,
),
obj_copy,
))
self.__worker_sem.release()
if EntropyCacher.STASHING_CACHE:
self.__stashing_cache[(key, cache_dir)] = obj_copy
except TypeError:
# sometimes, very rarely, copy.deepcopy() is unable
# to properly copy an object (blame Python bug)
sys.stdout.write("!!! cannot cache object with key %s\n" %
(key, ))
sys.stdout.flush()
#if const_debug_enabled():
# const_debug_write(__name__,
# "EntropyCacher.push, async_mode push %s, into %s" % (
# key, cache_dir,))
else:
#if const_debug_enabled():
# const_debug_write(__name__,
# "EntropyCacher.push, sync push %s, into %s" % (
# key, cache_dir,))
with self.__dump_data_lock:
entropy.dump.dumpobj(key, data, dump_dir=cache_dir)
def pop(self, key, cache_dir=None, aging_days=None):
"""
This is the place where data is retrieved from cache.
You must know the cache identifier used when push()
was called.
@param key: cache data identifier
@type key: string
@keyword cache_dir: alternative cache directory
@type cache_dir: string
@rtype: Python object
@return: object stored into the stack or None (if stack is empty)
"""
if cache_dir is None:
cache_dir = self.current_directory()
if EntropyCacher.STASHING_CACHE:
# object is being saved on disk, it's in RAM atm
ram_obj = self.__stashing_cache.get((key, cache_dir))
if ram_obj is not None:
return ram_obj
l_o = entropy.dump.loadobj
if not l_o:
return
return l_o(key, dump_dir=cache_dir, aging_days=aging_days)
@classmethod
def clear_cache_item(cls, cache_item, cache_dir=None):
"""
Clear Entropy Cache item from on-disk cache.
@param cache_item: Entropy Cache item identifier
@type cache_item: string
@keyword cache_dir: alternative cache directory
@type cache_dir: string
"""
if cache_dir is None:
cache_dir = cls.current_directory()
dump_path = os.path.join(cache_dir, cache_item)
dump_dir = os.path.dirname(dump_path)
for currentdir, subdirs, files in os.walk(dump_dir):
path = os.path.join(dump_dir, currentdir)
for item in files:
if item.endswith(entropy.dump.D_EXT):
item = os.path.join(path, item)
try:
os.remove(item)
except (
OSError,
IOError,
):
pass
try:
if not os.listdir(path):
os.rmdir(path)
except (
OSError,
IOError,
):
pass
def _graph_package(match, package, entropy_intf, show_complete = False,
quiet = False):
include_sys_pkgs = False
show_already_pulled_in = False
if show_complete:
include_sys_pkgs = True
show_already_pulled_in = True
graph = Graph()
stack = Lifo()
start_item = (match, package, None)
stack.push(start_item)
stack_cache = set()
# ensure package availability in graph, initialize now
graph.add(start_item, [])
depsorter = lambda x: entropy.dep.dep_getcpv(x[0])
while stack.is_filled():
item = stack.pop()
if item in stack_cache:
continue
stack_cache.add(item)
((pkg_id, repo_id,), _was_dep, dep_type) = item
# deps
repodb = entropy_intf.open_repository(repo_id)
deps = repodb.retrieveDependencies(pkg_id, extended = True,
resolve_conditional_deps = False)
graph_deps = []
for dep, x_dep_type in sorted(deps, key = depsorter):
if dep.startswith("!"): # conflict
continue
dep_item = entropy_intf.atom_match(dep)
if dep_item[0] == -1:
continue
do_include = True
if not include_sys_pkgs:
dep_repodb = entropy_intf.open_repository(dep_item[1])
do_include = not dep_repodb.isSystemPackage(dep_item[0])
g_item = (dep_item, dep, x_dep_type)
if do_include:
stack.push(g_item)
graph_deps.append(g_item)
graph.add(item, graph_deps)
def item_translation_func(match):
value = "%s" % (match[1],)
if match[2] is not None:
value += " %s%s%s" % (teal("{"), brown(str(match[2])), teal("}"),)
return value
_graph_to_stdout(entropy_intf, graph, graph.get_node(start_item),
item_translation_func, show_already_pulled_in, quiet)
if not quiet:
_show_graph_legend(entropy_intf)
show_dependencies_legend(entropy_intf)
del stack
graph.destroy()
return 0
def _revgraph_package(entropy_client, installed_pkg_id, package, dbconn,
show_complete = False, quiet = False):
include_sys_pkgs = False
show_already_pulled_in = False
include_build_deps = False
if show_complete:
include_sys_pkgs = True
show_already_pulled_in = True
include_build_deps = True
excluded_dep_types = [etpConst['dependency_type_ids']['bdepend_id']]
if not include_build_deps:
excluded_dep_types = None
graph = Graph()
stack = Lifo()
inst_item = (installed_pkg_id, package)
stack.push(inst_item)
stack_cache = set()
# ensure package availability in graph, initialize now
graph.add(inst_item, set())
def rev_pkgs_sorter(_pkg_id):
return dbconn.retrieveAtom(_pkg_id)
while stack.is_filled():
item = stack.pop()
if item in stack_cache:
continue
stack_cache.add(item)
pkg_id, _was_dep = item
rev_deps = dbconn.retrieveReverseDependencies(pkg_id,
exclude_deptypes = excluded_dep_types)
graph_deps = []
for rev_pkg_id in sorted(rev_deps, key = rev_pkgs_sorter):
dep = dbconn.retrieveAtom(rev_pkg_id)
do_include = True
if not include_sys_pkgs:
do_include = not dbconn.isSystemPackage(rev_pkg_id)
g_item = (rev_pkg_id, dep)
if do_include:
stack.push(g_item)
graph_deps.append(g_item)
graph.add(item, graph_deps)
def item_translation_func(match):
return match[1]
_graph_to_stdout(entropy_client, graph, graph.get_node(inst_item),
item_translation_func, show_already_pulled_in, quiet)
if not quiet:
_show_graph_legend(entropy_client)
del stack
graph.destroy()
return 0
class EntropyTransceiver(TextInterface):
"""
Base class for Entropy transceivers. This provides a common API across
all the available URI handlers.
How to use this class:
Let's consider that we have a valid EntropyUriHandler for ftp:// protocol
already installed via "add_uri_handler".
>> txc = EntropyTransceiver("ftp://*****:*****@myhost")
>> txc.set_speed_limit(150) # set speed limit to 150kb/sec
>> handler = txc.swallow()
>> handler.download("ftp://*****:*****@myhost/myfile.txt", "/tmp")
# download
"""
_URI_HANDLERS = []
@staticmethod
def add_uri_handler(entropy_uri_handler_class):
"""
Add custom URI handler to EntropyTransceiver class.
@param entropy_uri_handler_class: EntropyUriHandler based class
@type entropy_uri_handler_class; EntropyUriHandler instance
"""
if not issubclass(entropy_uri_handler_class, EntropyUriHandler):
raise AttributeError("EntropyUriHandler based class expected")
EntropyTransceiver._URI_HANDLERS.append(entropy_uri_handler_class)
@staticmethod
def remove_uri_handler(entropy_uri_handler_class):
"""
Remove custom URI handler to EntropyTransceiver class.
@param entropy_uri_handler_class: EntropyUriHandler based instance
@type entropy_uri_handler_class; EntropyUriHandler instance
@raise ValueError: if provided EntropyUriHandler is not in storage.
"""
if not issubclass(entropy_uri_handler_class, EntropyUriHandler):
raise AttributeError("EntropyUriHandler based class expected")
EntropyTransceiver._URI_HANDLERS.remove(entropy_uri_handler_class)
@staticmethod
def get_uri_handlers():
"""
Return a copy of the internal list of URI handler instances.
@return: URI handlers instances list
@rtype: list
"""
return EntropyTransceiver._URI_HANDLERS[:]
@staticmethod
def get_uri_name(uri):
"""
Given an URI, extract and return the URI name (hostname).
@param uri: URI to handle
@type uri: string
@return: URI name
@rtype: string
@raise UriHandlerNotFound: if no URI handlers can deal with given URI
string
"""
handlers = EntropyTransceiver.get_uri_handlers()
for handler in handlers:
if handler.approve_uri(uri):
return handler.get_uri_name(uri)
raise UriHandlerNotFound(
"no URI handler available for %s" % (uri,))
@staticmethod
def hide_sensible_data(uri):
"""
Given an URI, hide sensible data from string and return it back.
@param uri: URI to handle
@type uri: string
@return: URI cleaned
@rtype: string
@raise UriHandlerNotFound: if no URI handlers can deal with given URI
string
"""
handlers = EntropyTransceiver.get_uri_handlers()
for handler in handlers:
if handler.approve_uri(uri):
return handler.hide_sensible_data(uri)
raise UriHandlerNotFound(
"no URI handler available for %s" % (uri,))
def __init__(self, uri):
"""
EntropyTransceiver constructor, just pass the friggin URI(tm).
@param uri: URI to handle
@type uri: string
"""
self._uri = uri
self._speed_limit = 0
self._verbose = False
self._timeout = None
self._silent = None
self._output_interface = None
self.__with_stack = Lifo()
def __enter__(self):
"""
Support for "with" statement, this method will execute swallow() and
return a valid EntropyUriHandler instance.
"""
handler = self.swallow()
self.__with_stack.push(handler)
return handler
def __exit__(self, exc_type, exc_value, traceback):
"""
Support for "with" statement, this method will automagically close the
previously created EntropyUriHandler instance connection.
"""
handler = self.__with_stack.pop() # if this fails, it's not a good sign
handler.close()
def set_output_interface(self, output_interface):
"""
Provide alternative Entropy output interface (must be based on
entropy.output.TextInterface)
@param output_interface: new entropy.output.TextInterface instance to
use
@type output_interface: entropy.output.TextInterface based instance
@raise AttributeError: if argument passed is not correct
"""
if not isinstance(output_interface, TextInterface):
raise AttributeError(
"expected a valid TextInterface based instance")
self._output_interface = output_interface
def set_speed_limit(self, speed_limit):
"""
Set download/upload speed limit in kb/sec form.
Zero value will be considered as "disable speed limiter".
@param speed_limit: speed limit in kb/sec form.
@type speed_limit: int
@raise AttributeError: if speed_limit is not an integer
"""
if not const_isnumber(speed_limit):
raise AttributeError("expected a valid number")
self._speed_limit = speed_limit
def set_timeout(self, timeout):
"""
Set transceiver tx/rx timeout value in seconds.
@param timeout: timeout in seconds
@type timeout: int
"""
if not const_isnumber(timeout):
raise AttributeError("not a number")
self._timeout = timeout
def set_silent(self, silent):
"""
Disable transceivers verbosity.
@param verbosity: verbosity value
@type verbosity: bool
"""
self._silent = silent
def set_verbosity(self, verbosity):
"""
Set transceiver verbosity.
@param verbosity: verbosity value
@type verbosity: bool
"""
if not isinstance(verbosity, bool):
raise AttributeError("expected a valid bool")
self._verbose = verbosity
def swallow(self):
"""
Given the URI at the constructor, this method returns the first valid
URI handler instance found that can be used to do required action.
@raise entropy.exceptions.UriHandlerNotFound: when URI handler for given
URI is not available.
"""
handlers = EntropyTransceiver.get_uri_handlers()
for handler in handlers:
if handler.approve_uri(self._uri):
handler_instance = handler(self._uri)
if self._output_interface is not None:
handler_instance.set_output_interface(
self._output_interface)
if const_isnumber(self._speed_limit):
handler_instance.set_speed_limit(self._speed_limit)
handler_instance.set_verbosity(self._verbose)
handler_instance.set_silent(self._silent)
if const_isnumber(self._timeout):
handler_instance.set_timeout(self._timeout)
return handler_instance
raise UriHandlerNotFound(
"no URI handler available for %s" % (self._uri,))
def _graph_package(match, package, entropy_intf, show_complete = False,
quiet = False):
include_sys_pkgs = False
show_already_pulled_in = False
if show_complete:
include_sys_pkgs = True
show_already_pulled_in = True
graph = Graph()
stack = Lifo()
start_item = (match, package, None)
stack.push(start_item)
stack_cache = set()
# ensure package availability in graph, initialize now
graph.add(start_item, [])
depsorter = lambda x: entropy.dep.dep_getcpv(x[0])
while stack.is_filled():
item = stack.pop()
if item in stack_cache:
continue
stack_cache.add(item)
((pkg_id, repo_id,), _was_dep, dep_type) = item
# deps
repodb = entropy_intf.open_repository(repo_id)
deps = repodb.retrieveDependencies(pkg_id, extended = True,
resolve_conditional_deps = False)
graph_deps = []
for dep, x_dep_type in sorted(deps, key = depsorter):
if dep.startswith("!"): # conflict
continue
dep_item = entropy_intf.atom_match(dep)
if dep_item[0] == -1:
continue
do_include = True
if not include_sys_pkgs:
dep_repodb = entropy_intf.open_repository(dep_item[1])
do_include = not dep_repodb.isSystemPackage(dep_item[0])
g_item = (dep_item, dep, x_dep_type)
if do_include:
stack.push(g_item)
graph_deps.append(g_item)
graph.add(item, graph_deps)
def item_translation_func(match):
value = "%s" % (match[1],)
if match[2] is not None:
value += " %s%s%s" % (teal("{"), brown(str(match[2])), teal("}"),)
return value
_graph_to_stdout(entropy_intf, graph, graph.get_node(start_item),
item_translation_func, show_already_pulled_in, quiet)
if not quiet:
_show_graph_legend(entropy_intf)
show_dependencies_legend(entropy_intf)
del stack
graph.destroy()
return 0
class TopologicalSorter(object):
"""
This class implements the topological sorting algorithm presented by
R. E. Tarjan in 1972.
"""
def __init__(self, adjacency_map):
"""
TopologicalSorter constructor.
@param adjacency_map: dict form adjacency map
@type adjacency_map: dict
"""
object.__init__(self)
self.__adjacency_map = adjacency_map
self.__stack = Lifo()
def __topological_sort_visit_node(self, node, low, result):
"""
Internal method, visits a node ad push to stack.
"""
if node in low:
return
num = len(low)
low[node] = num
stack_pos = len(self.__stack)
self.__stack.push(node)
for successor in self.__adjacency_map[node]:
self.__topological_sort_visit_node(successor, low, result)
low[node] = min(low[node], low[successor])
if num == low[node]:
component = tuple()
while len(self.__stack) > stack_pos:
component += (self.__stack.pop(), )
result.append(component)
for item in component:
low[item] = len(self.__adjacency_map)
def __strongly_connected_nodes(self):
"""
Find the strongly connected nodes in a adjacency_map using
Tarjan's algorithm.
adjacency_map should be a dictionary mapping node names to
lists of successor nodes.
"""
result = []
low = {}
for node in self.__adjacency_map:
self.__topological_sort_visit_node(node, low, result)
return result
def __topological_sort(self, graph):
"""
Effectively executes topological sorting on given graph.
"""
# initialize count map
count = dict((node, 0) for node in graph)
for node in graph:
for successor in graph[node]:
count[successor] += 1
ready_stack = Lifo()
for node in graph:
if count[node] == 0:
ready_stack.push(node)
dep_level = 1
result = {}
while ready_stack.is_filled():
node = ready_stack.pop()
result[dep_level] = node
dep_level += 1
for successor in graph[node]:
count[successor] -= 1
if count[successor] == 0:
ready_stack.push(successor)
return result
def get_stored_adjacency_map(self):
"""
Return stored adjacency map used for sorting.
@return: stored adjacency map
@rtype: dict
"""
return self.__adjacency_map
def sort(self):
"""
Given an adjacency map, identify strongly connected nodes,
then perform a topological sort on them.
@return: sorted graph representation
@rtype: dict
"""
# clear stack
self.__stack.clear()
components = self.__strongly_connected_nodes()
node_component = {}
for component in components:
for node in component:
node_component[node] = component
component_graph = {}
for node in self.__adjacency_map:
node_c = node_component[node]
obj = component_graph.setdefault(node_c, [])
for successor in self.__adjacency_map[node]:
successor_c = node_component[successor]
if node_c != successor_c:
obj.append(successor_c)
return self.__topological_sort(component_graph)
class EntropyCacher(Singleton):
# Max number of cache objects written at once
_OBJS_WRITTEN_AT_ONCE = 250
# Number of seconds between cache writeback to disk
WRITEBACK_TIMEOUT = 5
# If True, in-ram cache will be used to mitigate
# concurrent push/pop executions with push() not
# yet able to write data to disk.
STASHING_CACHE = True
"""
Entropy asynchronous and synchronous cache writer
and reader. This class is a Singleton and contains
a thread doing the cache writes asynchronously, thus
it must be stopped before your application is terminated
calling the stop() method.
Sample code:
>>> # import module
>>> from entropy.cache import EntropyCacher
...
>>> # first EntropyCacher load, start it
>>> cacher = EntropyCacher()
>>> cacher.start()
...
>>> # now store something into its cache
>>> cacher.push('my_identifier1', [1, 2, 3])
>>> # now store something synchronously
>>> cacher.push('my_identifier2', [1, 2, 3], async = False)
...
>>> # now flush all the caches to disk, and make sure all
>>> # is written
>>> cacher.sync()
...
>>> # now fetch something from the cache
>>> data = cacher.pop('my_identifier1')
[1, 2, 3]
...
>>> # now discard all the cached (async) writes
>>> cacher.discard()
...
>>> # and stop EntropyCacher
>>> cacher.stop()
"""
class SemaphoreTimeScheduled(TimeScheduled):
def __init__(self, sem, *args, **kwargs):
self._sem = sem
TimeScheduled.__init__(self, *args, **kwargs)
def kill(self):
self._sem.release()
return TimeScheduled.kill(self)
def init_singleton(self):
"""
Singleton overloaded method. Equals to __init__.
This is the place where all the properties initialization
takes place.
"""
self.__copy = copy
self.__alive = False
self.__cache_writer = None
self.__cache_buffer = Lifo()
self.__stashing_cache = {}
self.__inside_with_stmt = 0
self.__dump_data_lock = threading.Lock()
self.__worker_sem = threading.Semaphore(0)
# this lock ensures that all the writes are hold while it's acquired
self.__enter_context_lock = threading.RLock()
def __enter__(self):
"""
When used with the with statement, pause cacher on-disk writes.
"""
self.__enter_context_lock.acquire()
self.__inside_with_stmt += 1
def __exit__(self, exc_type, exc_value, traceback):
"""
When used with the with statement, pause cacher on-disk writes.
"""
self.__inside_with_stmt -= 1
self.__enter_context_lock.release()
def __copy_obj(self, obj):
"""
Return a copy of an object done by the standard
library "copy" module.
@param obj: object to copy
@type obj: any Python object
@rtype: copied object
@return: copied object
"""
return self.__copy.deepcopy(obj)
def __cacher(self, run_until_empty=False, sync=False, _loop=False):
"""
This is where the actual asynchronous copy takes
place. __cacher runs on a different threads and
all the operations done by this are atomic and
thread-safe. It just loops over and over until
__alive becomes False.
"""
try:
if self.__inside_with_stmt != 0:
return
except AttributeError:
# interpreter shutdown
pass
# make sure our set delay is respected
try:
self.__cache_writer.set_delay(EntropyCacher.WRITEBACK_TIMEOUT)
except AttributeError:
# can be None
pass
# sleep if there's nothing to do
if _loop:
try:
# CANBLOCK
self.__worker_sem.acquire()
# we just consumed one acquire()
# that was dedicated to actual data,
# put it back
self.__worker_sem.release()
except AttributeError:
pass
def _commit_data(_massive_data):
for (key, cache_dir), data in _massive_data:
d_o = entropy.dump.dumpobj
if d_o is not None:
d_o(key, data, dump_dir=cache_dir)
while self.__alive or run_until_empty:
if const_debug_enabled():
const_debug_write(
__name__,
"EntropyCacher.__cacher: loop: %s, alive: %s, empty: %s" % (_loop, self.__alive, run_until_empty),
)
with self.__enter_context_lock:
massive_data = []
try:
massive_data_count = EntropyCacher._OBJS_WRITTEN_AT_ONCE
except AttributeError: # interpreter shutdown
break
while massive_data_count > 0:
if _loop:
# extracted an item from worker_sem
# call down() on the semaphore without caring
# can't sleep here because we're in a critical region
# holding __enter_context_lock
self.__worker_sem.acquire(False)
massive_data_count -= 1
try:
data = self.__cache_buffer.pop()
except (ValueError, TypeError):
# TypeError is when objects are being destroyed
break # stack empty
massive_data.append(data)
if not massive_data:
break
task = ParallelTask(_commit_data, massive_data)
task.name = "EntropyCacherCommitter"
task.daemon = not sync
task.start()
if sync:
task.join()
if const_debug_enabled():
const_debug_write(
__name__, "EntropyCacher.__cacher [%s], writing %s objs" % (task, len(massive_data))
)
if EntropyCacher.STASHING_CACHE:
for (key, cache_dir), data in massive_data:
try:
del self.__stashing_cache[(key, cache_dir)]
except (AttributeError, KeyError):
continue
del massive_data[:]
del massive_data
@classmethod
def current_directory(cls):
"""
Return the path to current EntropyCacher cache storage directory.
"""
return entropy.dump.D_DIR
def start(self):
"""
This is the method used to start the asynchronous cache
writer but also the whole cacher. If this method is not
called, the instance will always trash and cache write
request.
@return: None
"""
self.__cache_buffer.clear()
self.__cache_writer = EntropyCacher.SemaphoreTimeScheduled(
self.__worker_sem, EntropyCacher.WRITEBACK_TIMEOUT, self.__cacher, _loop=True
)
self.__cache_writer.daemon = True
self.__cache_writer.name = "EntropyCacheWriter"
self.__cache_writer.set_delay_before(True)
self.__cache_writer.start()
while not self.__cache_writer.isAlive():
continue
self.__alive = True
def is_started(self):
"""
Return whether start is called or not. This equals to
checking if the cacher is running, thus is writing cache
to disk.
@return: None
"""
return self.__alive
def stop(self):
"""
This method stops the execution of the cacher, which won't
accept cache writes anymore. The thread responsible of writing
to disk is stopped here and the Cacher will be back to being
inactive. A watchdog will avoid the thread to freeze the
call if the write buffer is overloaded.
@return: None
"""
self.__alive = False
if self.__cache_writer is not None:
self.__cache_writer.kill()
# make sure it unblocks
self.__worker_sem.release()
self.__cache_writer.join()
self.__cache_writer = None
self.sync()
def sync(self):
"""
This method can be called anytime and forces the instance
to flush all the cache writes queued to disk. If wait == False
a watchdog prevents this call to get stuck in case of write
buffer overloads.
"""
self.__cacher(run_until_empty=True, sync=True)
def discard(self):
"""
This method makes buffered cache to be discarded synchronously.
@return: None
"""
self.__cache_buffer.clear()
self.__stashing_cache.clear()
def save(self, key, data, cache_dir=None):
"""
Save data object to cache asynchronously and in any case.
This method guarantees that cached data is stored even if cacher
is not started. If data cannot be stored, IOError will be raised.
@param key: cache data identifier
@type key: string
@param data: picklable object
@type data: any picklable object
@keyword cache_dir: alternative cache directory
@type cache_dir: string
"""
if cache_dir is None:
cache_dir = self.current_directory()
try:
with self.__dump_data_lock:
entropy.dump.dumpobj(key, data, dump_dir=cache_dir, ignore_exceptions=False)
except (EOFError, IOError, OSError) as err:
raise IOError("cannot store %s to %s. err: %s" % (key, cache_dir, repr(err)))
def push(self, key, data, async=True, cache_dir=None):
def _revgraph_package(entropy_client, installed_pkg_id, package, dbconn,
show_complete = False, quiet = False):
include_sys_pkgs = False
show_already_pulled_in = False
include_build_deps = False
if show_complete:
include_sys_pkgs = True
show_already_pulled_in = True
include_build_deps = True
excluded_dep_types = [etpConst['dependency_type_ids']['bdepend_id']]
if not include_build_deps:
excluded_dep_types = None
graph = Graph()
stack = Lifo()
inst_item = (installed_pkg_id, package)
stack.push(inst_item)
stack_cache = set()
# ensure package availability in graph, initialize now
graph.add(inst_item, set())
def rev_pkgs_sorter(_pkg_id):
return dbconn.retrieveAtom(_pkg_id)
while stack.is_filled():
item = stack.pop()
if item in stack_cache:
continue
stack_cache.add(item)
pkg_id, _was_dep = item
rev_deps = dbconn.retrieveReverseDependencies(pkg_id,
exclude_deptypes = excluded_dep_types)
graph_deps = []
for rev_pkg_id in sorted(rev_deps, key = rev_pkgs_sorter):
dep = dbconn.retrieveAtom(rev_pkg_id)
do_include = True
if not include_sys_pkgs:
do_include = not dbconn.isSystemPackage(rev_pkg_id)
g_item = (rev_pkg_id, dep)
if do_include:
stack.push(g_item)
graph_deps.append(g_item)
graph.add(item, graph_deps)
def item_translation_func(match):
return match[1]
_graph_to_stdout(entropy_client, graph, graph.get_node(inst_item),
item_translation_func, show_already_pulled_in, quiet)
if not quiet:
_show_graph_legend(entropy_client)
del stack
graph.destroy()
return 0
class TopologicalSorter(object):
"""
This class implements the topological sorting algorithm presented by
R. E. Tarjan in 1972.
"""
def __init__(self, adjacency_map):
"""
TopologicalSorter constructor.
@param adjacency_map: dict form adjacency map
@type adjacency_map: dict
"""
object.__init__(self)
self.__adjacency_map = adjacency_map
self.__stack = Lifo()
def __topological_sort_visit_node(self, node, low, result):
"""
Internal method, visits a node ad push to stack.
"""
if node in low:
return
num = len(low)
low[node] = num
stack_pos = len(self.__stack)
self.__stack.push(node)
for successor in self.__adjacency_map[node]:
self.__topological_sort_visit_node(successor, low, result)
low[node] = min(low[node], low[successor])
if num == low[node]:
component = tuple()
while len(self.__stack) > stack_pos:
component += (self.__stack.pop(),)
result.append(component)
for item in component:
low[item] = len(self.__adjacency_map)
def __strongly_connected_nodes(self):
"""
Find the strongly connected nodes in a adjacency_map using
Tarjan's algorithm.
adjacency_map should be a dictionary mapping node names to
lists of successor nodes.
"""
result = []
low = {}
for node in self.__adjacency_map:
self.__topological_sort_visit_node(node, low, result)
return result
def __topological_sort(self, graph):
"""
Effectively executes topological sorting on given graph.
"""
# initialize count map
count = dict((node, 0) for node in graph)
for node in graph:
for successor in graph[node]:
count[successor] += 1
ready_stack = Lifo()
for node in graph:
if count[node] == 0:
ready_stack.push(node)
dep_level = 1
result = {}
while ready_stack.is_filled():
node = ready_stack.pop()
result[dep_level] = node
dep_level += 1
for successor in graph[node]:
count[successor] -= 1
if count[successor] == 0:
ready_stack.push(successor)
return result
def get_stored_adjacency_map(self):
"""
Return stored adjacency map used for sorting.
@return: stored adjacency map
@rtype: dict
"""
return self.__adjacency_map
def sort(self):
"""
Given an adjacency map, identify strongly connected nodes,
then perform a topological sort on them.
@return: sorted graph representation
@rtype: dict
"""
# clear stack
self.__stack.clear()
components = self.__strongly_connected_nodes()
node_component = {}
for component in components:
for node in component:
node_component[node] = component
component_graph = {}
for node in self.__adjacency_map:
node_c = node_component[node]
obj = component_graph.setdefault(node_c, [])
for successor in self.__adjacency_map[node]:
successor_c = node_component[successor]
if node_c != successor_c:
obj.append(successor_c)
return self.__topological_sort(component_graph)
