class Signal(object):
def __init__(self):
self.__slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self.__slots:
func, selfid = key
if selfid is not None:
func(self.__slots[key], *args, **kargs)
else:
func(*args, **kargs)
def __get_key(self, slot):
if hasattr(slot, 'im_func'):
return (slot.im_func, id(slot.im_self))
else:
return (slot, None)
def connect(self, slot):
key = self.__get_key(slot)
if hasattr(slot, 'im_func'):
self.__slots[key] = slot.im_self
else:
self.__slots[key] = slot
def disconnect(self, slot):
key = self.__get_key(slot)
if key in self.__slots:
self.__slots.pop(key)
def clear(self):
self.__slots.clear()
class Signal(object):
"""
A Signal is callable. When called, it calls all the callables in its slots.
"""
def __init__(self):
self._slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self._slots:
func, _ = key
func(self._slots[key], *args, **kargs)
def connect(self, slot):
"""
Slots must call this to register a callback method.
:param slot: callable
"""
key = (slot.im_func, id(slot.im_self))
self._slots[key] = slot.im_self
def disconnect(self, slot):
"""
They can also unregister their callbacks here.
:param slot: callable
"""
key = (slot.im_func, id(slot.im_self))
if key in self._slots:
self._slots.pop(key)
def clear(self):
"""
Clears all slots
"""
self._slots.clear()
class Signal(object):
"""
A Signal is callable. When called, it calls all the callables in its slots.
"""
def __init__(self):
self._slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self._slots:
func, _ = key
func(self._slots[key], *args, **kargs)
def connect(self, slot):
"""
Slots must call this to register a callback method.
:param slot: callable
"""
key = (slot.im_func, id(slot.im_self))
self._slots[key] = slot.im_self
def disconnect(self, slot):
"""
They can also unregister their callbacks here.
:param slot: callable
"""
key = (slot.im_func, id(slot.im_self))
if key in self._slots:
self._slots.pop(key)
def clear(self):
"""
Clears all slots
"""
self._slots.clear()
class Signal(object):
def __init__(self):
self.__slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self.__slots:
func, selfid = key
if selfid is not None:
func(self.__slots[key], *args, **kargs)
else:
func(*args, **kargs)
def __get_key(self, slot):
if hasattr(slot, 'im_func'):
return (slot.im_func, id(slot.im_self))
else:
return (slot, None)
def connect(self, slot):
key = self.__get_key(slot)
if hasattr(slot, 'im_func'):
self.__slots[key] = slot.im_self
else:
self.__slots[key] = slot
def disconnect(self, slot):
key = self.__get_key(slot)
if key in self.__slots:
self.__slots.pop(key)
def clear(self):
self.__slots.clear()
class Signal(object):
def __init__(self):
self.__slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self.__slots:
func, _ = key
func(self.__slots[key], *args, **kargs)
def connect(self, slot):
key = (slot.__func__, id(slot.__self__))
self.__slots[key] = slot.__self__
def disconnect(self, slot):
key = (slot.__func__, id(slot.__self__))
if key in self.__slots:
self.__slots.pop(key)
def clear(self):
self.__slots.clear()
## Sample usage:
#class Model(object):
# def __init__(self, value):
# self.__value = value
# self.changed = Signal()
#
# def set_value(self, value):
# self.__value = value
# self.changed() # Emit signal
#
# def get_value(self):
# return self.__value
#
#
#class View(object):
# def __init__(self, model):
# self.model = model
# model.changed.connect(self.model_changed)
#
# def model_changed(self):
# print "New value:", self.model.get_value()
#
#
#model = Model(10)
#view1 = View(model)
#view2 = View(model)
#view3 = View(model)
#
#model.set_value(20)
#
#del view1
#model.set_value(30)
#
#model.changed.clear()
#model.set_value(40)
### end of http://code.activestate.com/recipes/576477/ }}}
class ObjectPool(object):
"""
This class allows to fetch mvc model objects using their UUID.
This requires to model to have a property called "uuid". All
class inheriting from the base 'Model' class will have this.
If implementing a custom model, the UUID property is responsible
for the removal and addition to the pool when it changes values.
Also see the UUIDProperty descriptor for an example implementation.
We can use this to store complex relations between objects where
references to each other can be replaced with the UUID.
For a multi-threaded version see ThreadedObjectPool.
"""
def __init__(self, *args, **kwargs):
object.__init__(self)
self._objects = WeakValueDictionary()
def add_or_get_object(self, obj):
try:
self.add_object(obj, force=False, silent=False)
return obj
except KeyError:
return self.get_object(obj.uuid)
def add_object(self, obj, force=False, fail_on_duplicate=False):
if not obj.uuid in self._objects or force:
self._objects[obj.uuid] = obj
elif fail_on_duplicate:
raise KeyError(
"UUID %s is already taken by another object %s, cannot add object %s"
% (obj.uuid, self._objects[obj.uuid], obj))
else:
# Just change the objects uuid, will break refs, but
# it prevents issues with inherited properties etc.
logger.warning(
"A duplicate UUID was passed to an ObjectPool for a %s object."
% obj)
obj.uuid = get_new_uuid()
def change_all_uuids(self):
# first get a copy off all uuids & objects:
items = list(self._objects.items())
for uuid, obj in items: # @UnusedVariable
obj.uuid = get_new_uuid()
def remove_object(self, obj):
if obj.uuid in self._objects and self._objects[obj.uuid] == obj:
del self._objects[obj.uuid]
def get_object(self, uuid):
obj = self._objects.get(uuid, None)
return obj
def clear(self):
self._objects.clear()
class ObjectPool(object):
"""
This class allows to fetch mvc model objects using their UUID.
This requires to model to have a property called "uuid". All
class inheriting from the base 'Model' class will have this.
If implementing a custom model, the UUID property is responsible
for the removal and addition to the pool when it changes values.
Also see the UUIDPropIntel class for an example implementation.
We can use this to store complex relations between objects where
references to each other can be replaced with the UUID.
For a multi-threaded version see ThreadedObjectPool.
"""
def __init__(self, *args, **kwargs):
object.__init__(self)
self._objects = WeakValueDictionary()
def add_or_get_object(self, obj):
try:
self.add_object(obj, force=False, silent=False)
return obj
except KeyError:
return self.get_object(obj.uuid)
def add_object(self, obj, force=False, fail_on_duplicate=False):
if not obj.uuid in self._objects or force:
self._objects[obj.uuid] = obj
elif fail_on_duplicate:
raise KeyError, "UUID %s is already taken by another object %s, cannot add object %s" % (obj.uuid, self._objects[obj.uuid], obj)
else:
# Just change the objects uuid, will break refs, but
# it prevents issues with inherited properties etc.
logger.warning("A duplicate UUID was passed to an ObjectPool for a %s object." % obj)
obj.uuid = get_new_uuid()
def change_all_uuids(self):
# first get a copy off all uuids & objects:
items = self._objects.items()
for uuid, obj in items: # @UnusedVariable
obj.uuid = get_new_uuid()
def remove_object(self, obj):
if obj.uuid in self._objects and self._objects[obj.uuid] == obj:
del self._objects[obj.uuid]
def get_object(self, uuid):
obj = self._objects.get(uuid, None)
return obj
def clear(self):
self._objects.clear()
class factory:
"""package generator
does weakref caching per repository
:cvar child_class: callable to generate packages
"""
child_class = package
def __init__(self, parent_repo):
self._parent_repo = parent_repo
self._cached_instances = WeakValueDictionary()
def new_package(self, *args):
"""generate a new package instance"""
inst = self._cached_instances.get(args)
if inst is None:
inst = self._cached_instances[args] = self.child_class(self, *args)
return inst
def __call__(self, *args, **kwds):
return self.new_package(*args, **kwds)
def clear(self):
"""wipe the weakref cache of packages instances"""
self._cached_instances.clear()
def _get_metadata(self, *args):
"""Pulls metadata from the repo/cache/wherever.
Must be overridden in derivatives.
"""
raise NotImplementedError
def _update_metadata(self, *args):
"""Updates metadata in the repo/cache/wherever.
Must be overridden in derivatives.
"""
raise NotImplementedError
def __getstate__(self):
d = self.__dict__.copy()
del d['_cached_instances']
return d
def __setstate__(self, state):
self.__dict__ = state.copy()
self.__dict__['_cached_instances'] = WeakValueDictionary()
class DictRegistryBase(RegistryBase, MappingRegistryMixin):
"""
A registry class with dict-based item storage.
Note that the registry just owns a weak reference to the items.
"""
key_error_class = KeyError
def __init__(self):
self.mapping = WeakValueDictionary()
def perform_cache_clear(self):
self.mapping.clear()
def register(self, key, value):
"""
Registration.
If the given key has already existed, the function will raise a
KeyError.
"""
if key in self.mapping:
raise self.key_error_class(key)
self.mapping[key] = value
self.perform_register(key, value)
def perform_register(self, key, value):
pass
def unregister(self, key):
"""
Unregistration.
If the given key does not exist, the function will fail silently.
"""
self.mapping.pop(key, None)
self.perform_unregister(key)
def perform_unregister(self, key):
pass
def __getitem__(self, key):
"""
Shortcuts to access registered items.
"""
return self.mapping[key]
class Signal(object):
def __init__(self):
self.__slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self.__slots:
func, _ = key
func(self.__slots[key], *args, **kargs)
def connect(self, slot):
key = (slot.__func__, id(slot.__self__))
self.__slots[key] = slot.__self__
def disconnect(self, slot):
key = (slot.__func__, id(slot.__self__))
if key in self.__slots:
self.__slots.pop(key)
def clear(self):
self.__slots.clear()
class ThreadLocalEntityCache(local):
def __init__(self):
self.lock = Lock()
self._dict = WeakValueDictionary()
def __contains__(self, key):
return key in self._dict
def __getitem__(self, key):
return self._dict[key]
def get(self, key, default=None):
return self._dict.get(key, default)
def clear(self):
self._dict.clear()
def keys(self):
return self._dict.keys()
def update(self, key, value):
""" Extract, insert or remove a value for a given key.
"""
with self.lock:
if value is None:
# remove
try:
del self._dict[key]
except KeyError:
pass
else:
return None
elif callable(value):
try:
# extract
return self._dict[key]
except KeyError:
# construct and insert
new_value = value()
self._dict[key] = new_value
return new_value
else:
# insert or replace
self._dict[key] = value
return value
class Signal(object):
def __init__(self):
self.__slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self.__slots:
func, _ = key
func(self.__slots[key], *args, **kargs)
def connect(self, slot):
key = (slot.__func__, id(slot.__self__))
self.__slots[key] = slot.__self__
def disconnect(self, slot):
key = (slot.__func__, id(slot.__self__))
if key in self.__slots:
self.__slots.pop(key)
def clear(self):
self.__slots.clear()
class SpamManagerCatched:
def __init__(self):
self._catched = WeakValueDictionary()
self.name = None
def _get_spam(self, name):
self.name = name
if name in self._catched:
return self._catched[name]
else:
obj = SpamManager(name)
self._catched[name] = obj
return obj
def _clear(self):
self._catched.clear()
def __repr__(self):
return self.name
class ThreadLocalEntityCache(local):
def __init__(self):
self.lock = Lock()
self._dict = WeakValueDictionary()
def __contains__(self, key):
return key in self._dict
def __getitem__(self, key):
return self._dict[key]
def clear(self):
self._dict.clear()
def update(self, key, value):
""" Extract, insert or remove a value for a given key.
"""
with self.lock:
if value is None:
# remove
try:
del self._dict[key]
except KeyError:
pass
else:
return None
elif callable(value):
try:
# extract
return self._dict[key]
except KeyError:
# construct and insert
new_value = value()
self._dict[key] = new_value
return new_value
else:
# insert or replace
self._dict[key] = value
return value
class Signal(object):
"""
Signal slot object, used for handling events passed to the objects in the gui.
"""
def __init__(self):
self.__slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self.__slots:
func, _ = key
func(self.__slots[key], *args, **kargs)
def connect(self, slot):
key = (slot.im_func, id(slot.im_self))
self.__slots[key] = slot.im_self
def disconnect(self, slot):
key = (slot.im_func, id(slot.im_self))
if key in self.__slots:
self.__slots.pop(key)
def clear(self):
self.__slots.clear()
class Signal(object):
def __init__(self):
self.__slots = WeakValueDictionary()
# For keeping references to _FuncHost objects.
self.__funchosts = {}
def __call__(self, *args, **kargs):
for key in self.__slots:
func, _ = key
func(self.__slots[key], *args, **kargs)
def connect(self, slot):
if inspect.ismethod(slot):
key = (slot.im_func, id(slot.im_self))
self.__slots[key] = slot.im_self
else:
host = _FuncHost(slot)
self.connect(host.meth)
# We stick a copy in here just to keep the instance alive.
self.__funchosts[slot] = host
def disconnect(self, slot):
if inspect.ismethod(slot):
key = (slot.im_func, id(slot.im_self))
if key in self.__slots:
self.__slots.pop(key)
else:
if slot in self.__funchosts:
self.disconnect(self.__funchosts[slot].meth)
self.__funchosts.pop(slot)
def clear(self):
self.__slots.clear()
self.__funchosts.clear()
def clear(self):
self.queue.clear()
return WeakValueDictionary.clear(self)
class JsonCacheHandler:
"""
Each time Eos is initialized, it loads data from packed JSON
(disk cache) into memory data cache, and uses it to instantiate
objects, which are stored in in-memory weakref object cache.
Positional arguments:
diskCacheFolder -- folder where on-disk cache files are stored
name -- unique indentifier of cache, e.g. Eos instance name
logger -- logger to use for errors
"""
def __init__(self, diskCacheFolder, name, logger):
self._diskCacheFile = os.path.join(diskCacheFolder, '{}.json.bz2'.format(name))
self._logger = logger
# Initialize memory data cache
self.__typeDataCache = {}
self.__attributeDataCache = {}
self.__effectDataCache = {}
self.__modifierDataCache = {}
self.__fingerprint = None
# Initialize weakref object cache
self.__typeObjCache = WeakValueDictionary()
self.__attributeObjCache = WeakValueDictionary()
self.__effectObjCache = WeakValueDictionary()
self.__modifierObjCache = WeakValueDictionary()
# If cache doesn't exist, silently finish initialization
if not os.path.exists(self._diskCacheFile):
return
# Read JSON into local variable
try:
with bz2.BZ2File(self._diskCacheFile, 'r') as file:
jsonData = file.read().decode('utf-8')
data = json.loads(jsonData)
# If file doesn't exist, JSON load errors occur, or
# anything else bad happens, do not load anything
# and leave values as initialized
except:
msg = 'error during reading cache'
self._logger.error(msg, childName='cacheHandler')
# Load data into data cache, if no errors occurred
# during JSON reading/parsing
else:
self.__updateMemCache(data)
def getType(self, typeId):
try:
type_ = self.__typeObjCache[typeId]
except KeyError:
# We do str(int(id)) here because JSON dictionaries
# always have strings as key
jsonTypeId = str(int(typeId))
try:
data = self.__typeDataCache[jsonTypeId]
except KeyError as e:
raise TypeFetchError(typeId) from e
groupId, catId, duration, discharge, optimal, falloff, tracking, fittable, effects, attribs = data
type_ = Type(typeId=typeId,
groupId=groupId,
categoryId=catId,
durationAttributeId=duration,
dischargeAttributeId=discharge,
rangeAttributeId=optimal,
falloffAttributeId=falloff,
trackingSpeedAttributeId=tracking,
fittableNonSingleton=fittable,
attributes={attrId: attrVal for attrId, attrVal in attribs},
effects=tuple(self.getEffect(effectId) for effectId in effects))
self.__typeObjCache[typeId] = type_
return type_
def getAttribute(self, attrId):
try:
attribute = self.__attributeObjCache[attrId]
except KeyError:
jsonAttrId = str(int(attrId))
try:
data = self.__attributeDataCache[jsonAttrId]
except KeyError as e:
raise AttributeFetchError(attrId) from e
maxAttributeId, defaultValue, highIsGood, stackable = data
attribute = Attribute(attributeId=attrId,
maxAttributeId=maxAttributeId,
defaultValue=defaultValue,
highIsGood=highIsGood,
stackable=stackable)
self.__attributeObjCache[attrId] = attribute
return attribute
def getEffect(self, effectId):
try:
effect = self.__effectObjCache[effectId]
except KeyError:
jsonEffectId = str(int(effectId))
try:
data = self.__effectDataCache[jsonEffectId]
except KeyError as e:
raise EffectFetchError(effectId) from e
effCategoryId, isOffence, isAssist, fitChanceId, buildStatus, modifiers = data
effect = Effect(effectId=effectId,
categoryId=effCategoryId,
isOffensive=isOffence,
isAssistance=isAssist,
fittingUsageChanceAttributeId=fitChanceId,
buildStatus=buildStatus,
modifiers=tuple(self.getModifier(modifierId) for modifierId in modifiers))
self.__effectObjCache[effectId] = effect
return effect
def getModifier(self, modifierId):
try:
modifier = self.__modifierObjCache[modifierId]
except KeyError:
jsonModifierId = str(int(modifierId))
try:
data = self.__modifierDataCache[jsonModifierId]
except KeyError as e:
raise ModifierFetchError(modifierId) from e
state, context, srcAttrId, operator, tgtAttrId, location, filType, filValue = data
modifier = Modifier(modifierId=modifierId,
state=state,
context=context,
sourceAttributeId=srcAttrId,
operator=operator,
targetAttributeId=tgtAttrId,
location=location,
filterType=filType,
filterValue=filValue)
self.__modifierObjCache[modifierId] = modifier
return modifier
def getFingerprint(self):
"""Get disk cache fingerprint."""
return self.__fingerprint
def updateCache(self, data, fingerprint):
"""
Updates on-disk and memory caches.
Positional arguments:
data -- dictionary with data to update
fingerprint -- string with fingerprint
"""
# Make light version of data and add fingerprint
# to it
data = self.__stripData(data)
data['fingerprint'] = fingerprint
# Update disk cache
os.makedirs(os.path.dirname(self._diskCacheFile), mode=0o755, exist_ok=True)
with bz2.BZ2File(self._diskCacheFile, 'w') as file:
jsonData = json.dumps(data)
file.write(jsonData.encode('utf-8'))
# Update data cache; encode to JSON and decode back
# to make sure form of data is the same as after
# loading it from cache
data = json.loads(jsonData)
self.__updateMemCache(data)
def __stripData(self, data):
"""
Rework passed data, stripping dictionary
keys from it to reduce space needed to store it.
"""
slimData = {}
slimTypes = {}
for typeRow in data['types']:
typeId = typeRow['typeId']
slimTypes[typeId] = (typeRow['groupId'],
typeRow['categoryId'],
typeRow['durationAttributeId'],
typeRow['dischargeAttributeId'],
typeRow['rangeAttributeId'],
typeRow['falloffAttributeId'],
typeRow['trackingSpeedAttributeId'],
typeRow['fittableNonSingleton'],
tuple(typeRow['effects']), # List -> tuple
tuple(typeRow['attributes'].items())) # Dictionary -> tuple
slimData['types'] = slimTypes
slimAttribs = {}
for attrRow in data['attributes']:
attrId = attrRow['attributeId']
slimAttribs[attrId] = (attrRow['maxAttributeId'],
attrRow['defaultValue'],
attrRow['highIsGood'],
attrRow['stackable'])
slimData['attributes'] = slimAttribs
slimEffects = {}
for effectRow in data['effects']:
effectId = effectRow['effectId']
slimEffects[effectId] = (effectRow['effectCategory'],
effectRow['isOffensive'],
effectRow['isAssistance'],
effectRow['fittingUsageChanceAttributeId'],
effectRow['buildStatus'],
tuple(effectRow['modifiers'])) # List -> tuple
slimData['effects'] = slimEffects
slimModifiers = {}
for modifierRow in data['modifiers']:
modifierId = modifierRow['modifierId']
slimModifiers[modifierId] = (modifierRow['state'],
modifierRow['context'],
modifierRow['sourceAttributeId'],
modifierRow['operator'],
modifierRow['targetAttributeId'],
modifierRow['location'],
modifierRow['filterType'],
modifierRow['filterValue'])
slimData['modifiers'] = slimModifiers
return slimData
def __updateMemCache(self, data):
"""
Loads data into memory data cache.
Positional arguments:
data -- dictionary with data to load
"""
self.__typeDataCache = data['types']
self.__attributeDataCache = data['attributes']
self.__effectDataCache = data['effects']
self.__modifierDataCache = data['modifiers']
self.__fingerprint = data['fingerprint']
# Also clear object cache to make sure objects composed
# from old data are gone
self.__typeObjCache.clear()
self.__attributeObjCache.clear()
self.__effectObjCache.clear()
self.__modifierObjCache.clear()
class DispatchTree(object):
def __init__(self):
# core data
self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy())
self.nodes = WeakValueDictionary()
self.nodes[0] = self.root
self.pools = {}
self.renderNodes = {}
self.tasks = {}
self.rules = []
self.poolShares = {}
self.commands = {}
# deduced properties
self.nodeMaxId = 0
self.poolMaxId = 0
self.renderNodeMaxId = 0
self.taskMaxId = 0
self.commandMaxId = 0
self.poolShareMaxId = 0
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
# listeners
self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange)
self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange)
# # JSA
# self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange)
self.renderNodeListener = ObjectListener(
self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange
)
self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange)
self.commandListener = ObjectListener(
onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange
)
self.poolShareListener = ObjectListener(self.onPoolShareCreation)
self.modifiedNodes = []
def registerModelListeners(self):
BaseNode.changeListeners.append(self.nodeListener)
Task.changeListeners.append(self.taskListener)
TaskGroup.changeListeners.append(self.taskListener)
RenderNode.changeListeners.append(self.renderNodeListener)
Pool.changeListeners.append(self.poolListener)
Command.changeListeners.append(self.commandListener)
PoolShare.changeListeners.append(self.poolShareListener)
def destroy(self):
BaseNode.changeListeners.remove(self.nodeListener)
Task.changeListeners.remove(self.taskListener)
RenderNode.changeListeners.remove(self.renderNodeListener)
Pool.changeListeners.remove(self.poolListener)
Command.changeListeners.remove(self.commandListener)
PoolShare.changeListeners.remove(self.poolShareListener)
self.root = None
self.nodes.clear()
self.pools.clear()
self.renderNodes.clear()
self.tasks.clear()
self.rules = None
self.commands.clear()
self.poolShares = None
self.modifiedNodes = None
self.toCreateElements = None
self.toModifyElements = None
self.toArchiveElements = None
def findNodeByPath(self, path, default=None):
nodenames = splitpath(path)
node = self.root
for name in nodenames:
for child in node.children:
if child.name == name:
node = child
break
else:
return default
return node
def updateCompletionAndStatus(self):
self.root.updateCompletionAndStatus()
def validateDependencies(self):
nodes = set()
for dependency in self.modifiedNodes:
for node in dependency.reverseDependencies:
nodes.add(node)
del self.modifiedNodes[:]
for node in nodes:
# logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) )
if not hasattr(node, "task") or node.task is None:
continue
if isinstance(node, TaskNode):
if node.checkDependenciesSatisfaction():
for cmd in node.task.commands:
if cmd.status == CMD_BLOCKED:
cmd.status = CMD_READY
else:
for cmd in node.task.commands:
if cmd.status == CMD_READY:
cmd.status = CMD_BLOCKED
# TODO: may be needed to check dependencies on task groups
# so far, a hack is done on the client side when submitting:
# dependencies of a taksgroup are reported on each task of its heirarchy
#
# elif isinstance(node, FolderNode):
#
# if node.checkDependenciesSatisfaction():
# for cmd in node.getAllCommands():
# if cmd.status == CMD_BLOCKED:
# cmd.status = CMD_READY
# else:
# for cmd in node.getAllCommands():
# if cmd.status == CMD_READY:
# cmd.status = CMD_BLOCKED
def registerNewGraph(self, graph):
user = graph["user"]
taskDefs = graph["tasks"]
poolName = graph["poolName"]
if "maxRN" in graph.items():
maxRN = int(graph["maxRN"])
else:
maxRN = -1
#
# Create objects.
#
tasks = [None for i in xrange(len(taskDefs))]
for (index, taskDef) in enumerate(taskDefs):
if taskDef["type"] == "Task":
# logger.debug("taskDef.watcherPackages = %s" % taskDef["watcherPackages"])
# logger.debug("taskDef.runnerPackages = %s" % taskDef["runnerPackages"])
task = self._createTaskFromJSON(taskDef, user)
elif taskDef["type"] == "TaskGroup":
task = self._createTaskGroupFromJSON(taskDef, user)
tasks[index] = task
root = tasks[graph["root"]]
# get the pool
try:
pool = self.pools[poolName]
except KeyError:
pool = Pool(None, poolName)
self.pools[poolName] = pool
#
# Rebuild full job hierarchy
#
for (taskDef, task) in zip(taskDefs, tasks):
if taskDef["type"] == "TaskGroup":
for taskIndex in taskDef["tasks"]:
task.addTask(tasks[taskIndex])
tasks[taskIndex].parent = task
#
# Compute dependencies for each created task or taskgroup object.
#
dependencies = {}
for (taskDef, task) in zip(taskDefs, tasks):
taskDependencies = {}
if not isinstance(taskDef["dependencies"], list):
raise SyntaxError(
"Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef["dependencies"]
)
if not all(
(
(isinstance(i, int) and isinstance(sl, list) and all((isinstance(s, int) for s in sl)))
for (i, sl) in taskDef["dependencies"]
)
):
raise SyntaxError(
"Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef["dependencies"]
)
for (taskIndex, statusList) in taskDef["dependencies"]:
taskDependencies[tasks[taskIndex]] = statusList
dependencies[task] = taskDependencies
#
# Apply rules to generate dispatch tree nodes.
#
if not self.rules:
logger.warning("graph submitted but no rule has been defined")
unprocessedTasks = [root]
nodes = []
while unprocessedTasks:
unprocessedTask = unprocessedTasks.pop(0)
for rule in self.rules:
try:
nodes += rule.apply(unprocessedTask)
except RuleError:
logger.warning("rule %s failed for graph %s" % (rule, graph))
raise
if isinstance(unprocessedTask, TaskGroup):
for task in unprocessedTask:
unprocessedTasks.append(task)
# create the poolshare, if any, and affect it to the node
if pool:
# FIXME nodes[0] may not be the root node of the graph...
ps = PoolShare(None, pool, nodes[0], maxRN)
# if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true
if maxRN != -1:
ps.userDefinedMaxRN = True
#
# Process dependencies
#
for rule in self.rules:
rule.processDependencies(dependencies)
for node in nodes:
assert isinstance(node.id, int)
self.nodes[node.id] = node
# Init number of command in hierarchy
self.populateCommandCounts(nodes[0])
return nodes
def populateCommandCounts(self, node):
"""
Updates "commandCount" over a whole hierarchy starting from the given node.
"""
res = 0
if isinstance(node, FolderNode):
for child in node.children:
res += self.populateCommandCounts(child)
elif isinstance(node, TaskNode):
res = len(node.task.commands)
node.commandCount = res
return res
def _createTaskGroupFromJSON(self, taskGroupDefinition, user):
# name, parent, arguments, environment, priority, dispatchKey, strategy
id = None
name = taskGroupDefinition["name"]
parent = None
arguments = taskGroupDefinition["arguments"]
environment = taskGroupDefinition["environment"]
requirements = taskGroupDefinition["requirements"]
maxRN = taskGroupDefinition["maxRN"]
priority = taskGroupDefinition["priority"]
dispatchKey = taskGroupDefinition["dispatchKey"]
strategy = taskGroupDefinition["strategy"]
strategy = loadStrategyClass(strategy.encode())
strategy = strategy()
tags = taskGroupDefinition["tags"]
timer = None
if "timer" in taskGroupDefinition.keys():
timer = taskGroupDefinition["timer"]
return TaskGroup(
id,
name,
parent,
user,
arguments,
environment,
requirements,
maxRN,
priority,
dispatchKey,
strategy,
tags=tags,
timer=timer,
)
def _createTaskFromJSON(self, taskDefinition, user):
# id, name, parent, user, priority, dispatchKey, runner, arguments,
# validationExpression, commands, requirements=[], minNbCores=1,
# maxNbCores=0, ramUse=0, environment={}
name = taskDefinition["name"]
runner = taskDefinition["runner"]
arguments = taskDefinition["arguments"]
environment = taskDefinition["environment"]
requirements = taskDefinition["requirements"]
maxRN = taskDefinition["maxRN"]
priority = taskDefinition["priority"]
dispatchKey = taskDefinition["dispatchKey"]
validationExpression = taskDefinition["validationExpression"]
minNbCores = taskDefinition["minNbCores"]
maxNbCores = taskDefinition["maxNbCores"]
ramUse = taskDefinition["ramUse"]
lic = taskDefinition["lic"]
tags = taskDefinition["tags"]
runnerPackages = taskDefinition.get("runnerPackages", "")
watcherPackages = taskDefinition.get("watcherPackages", "")
timer = None
if "timer" in taskDefinition.keys():
timer = taskDefinition["timer"]
maxAttempt = taskDefinition.get("maxAttempt", 1)
task = Task(
None,
name,
None,
user,
maxRN,
priority,
dispatchKey,
runner,
arguments,
validationExpression,
[],
requirements,
minNbCores,
maxNbCores,
ramUse,
environment,
lic=lic,
tags=tags,
timer=timer,
maxAttempt=maxAttempt,
runnerPackages=runnerPackages,
watcherPackages=watcherPackages,
)
for commandDef in taskDefinition["commands"]:
description = commandDef["description"]
arguments = commandDef["arguments"]
cmd = Command(
None, description, task, arguments, runnerPackages=runnerPackages, watcherPackages=watcherPackages
)
task.commands.append(cmd)
# import sys
# logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd)))
return task
## Resets the lists of elements to create or update in the database.
#
def resetDbElements(self):
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
## Recalculates the max ids of all elements. Generally called after a reload from db.
#
def recomputeMaxIds(self):
self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0
self.nodeMaxId = max(self.nodeMaxId, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId())
self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0
self.poolMaxId = max(self.poolMaxId, StatDB.getPoolsMaxId())
self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0
self.renderNodeMaxId = max(self.renderNodeMaxId, StatDB.getRenderNodesMaxId())
self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0
self.taskMaxId = max(self.taskMaxId, StatDB.getTasksMaxId(), StatDB.getTaskGroupsMaxId())
self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0
self.commandMaxId = max(self.commandMaxId, StatDB.getCommandsMaxId())
self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0
self.poolShareMaxId = max(self.poolShareMaxId, StatDB.getPoolSharesMaxId())
## Removes from the dispatchtree the provided element and all its parents and children.
#
def unregisterElementsFromTree(self, element):
# /////////////// Handling of the Task
if isinstance(element, Task):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for cmd in element.commands:
self.unregisterElementsFromTree(cmd)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskGroup
elif isinstance(element, TaskGroup):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for task in element.tasks:
self.unregisterElementsFromTree(task)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskNode
elif isinstance(element, TaskNode):
# remove the element from the children of the parent
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the FolderNode
elif isinstance(element, FolderNode):
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the Command
elif isinstance(element, Command):
del self.commands[element.id]
self.toArchiveElements.append(element)
### methods called after interaction with a Task
def onTaskCreation(self, task):
# logger.info(" -- on task creation: %s" % task)
if task.id is None:
self.taskMaxId += 1
task.id = self.taskMaxId
self.toCreateElements.append(task)
else:
self.taskMaxId = max(self.taskMaxId, task.id, StatDB.getTasksMaxId(), StatDB.getTaskGroupsMaxId())
self.tasks[task.id] = task
def onTaskDestruction(self, task):
# logger.info(" -- on task destruction: %s" % task)
self.unregisterElementsFromTree(task)
def onTaskChange(self, task, field, oldvalue, newvalue):
"""
Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes
However in order to keep track of comments (stored in task's tags[comment] field), we make the following change:
- enable task/taskgroups update in DB (cf pulidb.py)
- disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs
BUT the only field we want to update: "tags"
"""
if field == "tags":
self.toModifyElements.append(task)
### methods called after interaction with a BaseNode
def onNodeCreation(self, node):
# logger.info(" -- on node creation: %s" % node)
if node.id is None:
self.nodeMaxId += 1
node.id = self.nodeMaxId
self.toCreateElements.append(node)
else:
self.nodeMaxId = max(self.nodeMaxId, node.id, StatDB.getFolderNodesMaxId(), StatDB.getTaskNodesMaxId())
if node.parent is None:
node.parent = self.root
def onNodeDestruction(self, node):
# logger.info(" -- on node destruction: %s" % node)
del self.nodes[node.id]
def onNodeChange(self, node, field, oldvalue, newvalue):
# logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) )
# FIXME: do something when nodes are reparented from or to the root node
if node.id is not None:
self.toModifyElements.append(node)
if field == "status" and node.reverseDependencies:
self.modifiedNodes.append(node)
### methods called after interaction with a RenderNode
def onRenderNodeCreation(self, renderNode):
if renderNode.id is None:
self.renderNodeMaxId += 1
renderNode.id = self.renderNodeMaxId
self.toCreateElements.append(renderNode)
else:
self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id, StatDB.getRenderNodesMaxId())
self.renderNodes[renderNode.name] = renderNode
def onRenderNodeDestruction(self, rendernode):
try:
del self.renderNodes[rendernode.name]
self.toArchiveElements.append(rendernode)
except KeyError:
# TOFIX: use of class method vs obj method in changeListener might generate a duplicate call
logger.warning("RN %s seems to have been deleted already." % rendernode.name)
def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue):
if field == "performance":
self.toModifyElements.append(rendernode)
### methods called after interaction with a Pool
def onPoolCreation(self, pool):
if pool.id is None:
self.poolMaxId += 1
pool.id = self.poolMaxId
self.toCreateElements.append(pool)
else:
self.poolMaxId = max(self.poolMaxId, pool.id, StatDB.getPoolsMaxId())
self.pools[pool.name] = pool
def onPoolDestruction(self, pool):
del self.pools[pool.name]
self.toArchiveElements.append(pool)
def onPoolChange(self, pool, field, oldvalue, newvalue):
if pool not in self.toModifyElements:
self.toModifyElements.append(pool)
### methods called after interaction with a Command
def onCommandCreation(self, command):
if command.id is None:
self.commandMaxId += 1
command.id = self.commandMaxId
self.toCreateElements.append(command)
else:
self.commandMaxId = max(self.commandMaxId, command.id, StatDB.getCommandsMaxId())
self.commands[command.id] = command
def onCommandChange(self, command, field, oldvalue, newvalue):
self.toModifyElements.append(command)
if command.task is not None:
for node in command.task.nodes.values():
node.invalidate()
### methods called after interaction with a Pool
def onPoolShareCreation(self, poolShare):
if poolShare.id is None:
self.poolShareMaxId += 1
poolShare.id = self.poolShareMaxId
self.toCreateElements.append(poolShare)
else:
self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id, StatDB.getPoolSharesMaxId())
self.poolShares[poolShare.id] = poolShare
class TaskQueue(AbstractTaskQueue):
"""Simple in-memory task queue implementation"""
@classmethod
def factory(cls, url, name=const.DEFAULT, *args, **kw):
obj = _REFS.get((url, name))
if obj is None:
obj = _REFS[(url, name)] = cls(url, name, *args, **kw)
return obj
def __init__(self, *args, **kw):
super(TaskQueue, self).__init__(*args, **kw)
self.queue = Queue()
self.results = WeakValueDictionary()
self.results_lock = Lock()
def _init_result(self, result, status, message):
with self.results_lock:
if result.id in self.results:
return False
self.results[result.id] = result
result.__status = status
result.__value = Queue()
result.__task = message
result.__args = {}
result.__lock = Lock()
result.__for = None
return True
def enqueue_task(self, result, message):
if self._init_result(result, const.ENQUEUED, message):
self.queue.put(result)
return True
return False
def defer_task(self, result, message, args):
if self._init_result(result, const.PENDING, message):
results = self.results
# keep references to results to prevent GC
result.__refs = [results.get(arg) for arg in args]
return True
return False
def undefer_task(self, task_id):
result = self.results[task_id]
self.queue.put(result)
def get(self, timeout=None):
try:
result = self.queue.get(timeout=timeout)
except Empty:
return None
result.__status = const.PROCESSING
return result.id, result.__task
def size(self):
return len(self.results)
def discard_pending(self):
with self.results_lock:
while True:
try:
self.queue.get_nowait()
except Empty:
break
self.results.clear()
def reserve_argument(self, argument_id, deferred_id):
result = self.results.get(argument_id)
if result is None:
return (False, None)
with result.__lock:
if result.__for is not None:
return (False, None)
result.__for = deferred_id
try:
message = result.__value.get_nowait()
except Empty:
message = None
if message is not None:
with self.results_lock:
self.results.pop(argument_id, None)
return (True, message)
def set_argument(self, task_id, argument_id, message):
result = self.results[task_id]
with self.results_lock:
self.results.pop(argument_id, None)
with result.__lock:
result.__args[argument_id] = message
return len(result.__args) == len(result.__refs)
def get_arguments(self, task_id):
try:
return self.results[task_id].__args
except KeyError:
return {}
def set_task_timeout(self, task_id, timeout):
pass
def get_status(self, task_id):
result = self.results.get(task_id)
return None if result is None else result.__status
def set_result(self, task_id, message, timeout):
result = self.results.get(task_id)
if result is not None:
with result.__lock:
result.__value.put(message)
return result.__for
def pop_result(self, task_id, timeout):
result = self.results.get(task_id)
if result is None:
return const.TASK_EXPIRED
# with result.__lock:
# if result.__for is not None:
# raise NotImplementedError
# #return const.RESERVED
# result.__for = task_id
try:
if timeout == 0:
value = result.__value.get_nowait()
else:
value = result.__value.get(timeout=timeout)
except Empty:
value = None
else:
self.results.pop(task_id, None)
return value
def discard_result(self, task_id, task_expired_token):
result = self.results.pop(task_id)
if result is not None:
result.__value.put(task_expired_token)
class Signal(object):
def __init__(self):
self.__slots = WeakValueDictionary()
def __call__(self, *args, **kargs):
for key in self.__slots:
func, _ = key
func(self.__slots[key], *args, **kargs)
def connect(self, slot):
if PYTHON3:
key = (slot.__func__, id(slot.__self__))
self.__slots[key] = slot.__self__
else:
key = (slot.im_func, id(slot.im_self))
self.__slots[key] = slot.im_self
def disconnect(self, slot):
if PYTHON3:
key = (slot.__func__, id(slot.__self__))
if key in self.__slots:
self.__slots.pop(key)
else:
key = (slot.im_func, id(slot.im_self))
if key in self.__slots:
self.__slots.pop(key)
def clear(self):
self.__slots.clear()
## Sample usage:
#class Model(object):
# def __init__(self, value):
# self.__value = value
# self.changed = Signal()
#
# def set_value(self, value):
# self.__value = value
# self.changed() # Emit signal
#
# def get_value(self):
# return self.__value
#
#
#class View(object):
# def __init__(self, model):
# self.model = model
# model.changed.connect(self.model_changed)
#
# def model_changed(self):
# print "New value:", self.model.get_value()
#
#
#model = Model(10)
#view1 = View(model)
#view2 = View(model)
#view3 = View(model)
#
#model.set_value(20)
#
#del view1
#model.set_value(30)
#
#model.changed.clear()
#model.set_value(40)
### end of http://code.activestate.com/recipes/576477/ }}}
class CommandDispatcher(object):
count = 0
def __init__(self, dispatcher_id=None):
try:
self.dispatcher_id = dispatcher_id
except AttributeError:
pass
self.main_window = None
""":type: QtGui.QWidget"""
self._undo_stack = UndoStack()
self._action_history = []
self._actions = {}
self._commands = {}
self._parent_dispatcher = None
""":type: ref[CommandDispatcher]"""
self._children_dispatchers = WeakValueDictionary()
self._action_added = MrSignal()
self._main_data = None
self.main_data_can_change = True
@property
def main_data(self):
return self._main_data
@main_data.setter
def main_data(self, value):
if self.main_data_can_change:
self._main_data = value
@property
def undo_stack(self):
try:
return self._parent_dispatcher().undo_stack
except (TypeError, AttributeError):
return self._undo_stack
@property
def action_history(self):
try:
return self._parent_dispatcher().action_history
except (TypeError, AttributeError):
return self._action_history
@property
def action_added(self):
try:
return self._parent_dispatcher().action_added
except (TypeError, AttributeError):
return self._action_added
def clear_children(self):
self._children_dispatchers.clear()
def add_child(self, dispatcher):
if dispatcher.dispatcher_id is None:
CommandDispatcher.count += 1
dispatcher.dispatcher_id = str(CommandDispatcher.count)
if dispatcher.dispatcher_id in self._children_dispatchers:
assert self._children_dispatchers[dispatcher.dispatcher_id] is dispatcher
return
self._children_dispatchers[dispatcher.dispatcher_id] = dispatcher
dispatcher.set_parent(self)
def set_parent(self, parent_dispatcher):
self._parent_dispatcher = ref(parent_dispatcher)
def multiple_dispatch(self, actions):
for action in actions:
self.dispatch(action)
def _get_command(self, action):
if isinstance(action, Action):
action_name = action.action_name
try:
command = self._commands[action_name](action, main_window=self.main_window)
except KeyError:
raise TypeError('CommandDispatcher4: Command %s not found in defined actions!' % str(action_name))
elif isinstance(action, (Command, ChildCommand)):
command = action
command.main_window = self.main_window
else:
raise TypeError('CommandDispatcher4: Action type not valid! %s' % str(action))
return command
def _try_undo_redo(self, action):
if isinstance(action, str):
tmp = action.upper()
if tmp == 'UNDO':
self.action_history.append('Undo')
self.action_added.emit(action)
self.undo_stack.undo()
return True
elif tmp == 'REDO':
self.action_history.append('Redo')
self.action_added.emit(action)
self.undo_stack.redo()
return True
return False
def undo(self):
self.dispatch('Undo')
def redo(self):
self.dispatch('Redo')
def _subdata(self, data):
return self.main_data.subdata(data)
def _action_str(self, action, action_data=None):
assert isinstance(action, str)
if action.upper() in ('REDO', 'UNDO'):
return action
# debuginfo(self.dispatcher_id, self._parent_dispatcher)
if None not in (self.dispatcher_id, self._parent_dispatcher):
if action_data is not None:
data = action_data.split()
# debuginfo(22222, data)
if data[0] is None and data[1] is None:
action = '%s.%s()' % (self.dispatcher_id, action)
elif data[0] is None:
action = '%s.%s%s' % (self.dispatcher_id, action, data[1])
elif data[1] is None:
action = '%s[%s].%s()' % (self.dispatcher_id, data[0], action)
else:
action = '%s[%s].%s%s' % (self.dispatcher_id, data[0], action, data[1])
else:
action = '%s.%s' % (self.dispatcher_id, action)
return action
def dispatch(self, action, tracking=True):
if self._try_undo_redo(action):
return True
try:
action_name, action_data = action
action_name = action_name.replace('.', '_')
except (TypeError, ValueError):
assert isinstance(action, str)
action_info = self.parse_action(action)
# debuginfo(action_info)
return self._dispatch(action_info, tracking, action)
if not isinstance(action_data, tuple):
action_data = (action_data,)
try:
action_cls = self._actions[action_name]
except KeyError:
raise TypeError('CommandDispatcher4: Action type not valid! %s' % str(action_name))
action_data = action_cls.ActionDataCls(*action_data)
try:
# debuginfo('getting action_str')
action_str = self._action_str(action_name, action_data)
# debuginfo(1111111, action_str)
return self._parent_dispatcher()._traceback(action_str, tracking, action_data)
except (TypeError, AttributeError):
action_str = '%s%s' % (action_name, str(action_data))
action_info = self.parse_action(action_str)
return self._dispatch(action_info, tracking, action_str, action_data)
def _traceback(self, action, tracking=True, action_data=None):
action_str = self._action_str(action)
try:
return self._parent_dispatcher()._traceback(action_str, tracking, action_data)
except (TypeError, AttributeError):
action_info = self.parse_action(action_str)
# debuginfo(action_str)
return self._dispatch(action_info, tracking, action_str, action_data)
def _dispatch(self, action_info, tracking, action_str, action_data=None):
_dispatches, _action = action_info
# debuginfo(action_info)
if len(_dispatches) == 0:
action_name, action_data_ = _action
if action_data is None:
action_data = action_data_
# if not isinstance(action_data, tuple):
# action_data = (action_data,)
return self._final_dispatch(action_name, action_data, action_str, tracking)
dispatcher_id, dispatcher_data = _dispatches[0]
# debuginfo(action_info)
if self.dispatcher_id is not None:
try:
assert dispatcher_id == self.dispatcher_id
except AssertionError:
print('This dispatcher = %s, other dispatcher = %s' % (self.dispatcher_id, dispatcher_id))
raise
try:
dispatcher_id = _dispatches[1][0]
except IndexError:
_action_info = [], _action
return self._dispatch(_action_info, tracking, action_str, action_data)
else:
return self._children_dispatchers[dispatcher_id]._dispatch(action_info, tracking, action_str, action_data)
# debuginfo(self.dispatcher_id, list(self._children_dispatchers.keys()))
dispatcher = self._children_dispatchers[dispatcher_id]
# FIXME: should the dispatcher be responsible for this? might be taken care of by the commands
if dispatcher_data is not None:
subdata = self._subdata(dispatcher_data)
else:
subdata = None
old_main_data = dispatcher.get_model
if subdata is not None:
dispatcher.get_model = subdata
_action_info = _dispatches[1:], _action
# noinspection PyProtectedMember
dispatch_result = dispatcher._dispatch(_action_info, tracking, action_str, action_data)
dispatcher.get_model = old_main_data
return dispatch_result
def _final_dispatch(self, action_name, action_data, action_str, tracking=True):
# debuginfo(action_str)
if self._try_undo_redo(action_name):
return True
try:
action_cls = self._actions[action_name]
except KeyError:
raise TypeError('CommandDispatcher4: Action type not valid! %s' % str(action_name))
if isinstance(action_data, tuple):
action_data = action_cls.ActionDataCls(*action_data)
assert isinstance(action_data, action_cls.ActionDataCls)
action = action_cls(action_data)
action.get_model = self.main_data
command = self._get_command(action)
if command is None:
return False
command = self._wrap_command(command)
command.skip_first = False
command.redo()
command_result = command.command_result
command.skip_first = True
# TODO: not sure if this is the desired behavior
if command_result is False:
command.finalize()
return False
action_ = command.action
if action_.log_action is True and tracking is True:
if action_str is None:
action_str = str(action_)
self.action_history.append(action_str)
# this notifies the main window that an action has been added, so that it can update the log
self.action_added.emit(action_str)
# if the action is successful, push it to the stack (it will be skipped on first push)
if command_result is True:
if command.push_to_stack and tracking is True:
self.undo_stack.push(command)
if command.set_clean is True:
self.undo_stack.setClean()
return True
else:
return False
def _wrap_command(self, command):
try:
return self._parent_dispatcher()._wrap_command(command)
except (TypeError, AttributeError):
return command
def verify(self):
action_keys = set(self._actions.keys())
command_keys = set(self._commands.keys())
if action_keys != command_keys:
if len(action_keys) > len(command_keys):
raise Exception("CommandDispatcher4: Missing commands! %s" % str(action_keys - command_keys))
else:
raise Exception("CommandDispatcher4: Missing actions! %s" % str(command_keys - action_keys))
for key, child in iteritems(self._children_dispatchers):
child.verify()
def finalize(self):
self._parent_dispatcher = None
self._actions.clear()
self._commands.clear()
for key, child in iteritems(self._children_dispatchers):
child.finalize()
self._children_dispatchers.clear()
def __call__(self, action_name):
action_name = action_name.replace('.', '_')
def add_action(cls):
if issubclass(cls, Action):
self._actions[action_name] = cls
elif issubclass(cls, Command):
self._commands[action_name] = cls
else:
raise TypeError("CommandDispatcher4: %s is not an Action or Command!" % cls.__name__)
cls.action_name = action_name
return cls
return add_action
@staticmethod
def parse_action(s):
# debuginfo(s)
tmp = s
data = ''
if s[-1] == ')':
count = 1
for i in range(1, len(s)):
a = s[-i-1]
if a == ')':
count += 1
elif a == '(':
count -= 1
if count == 0:
j = len(s) - i - 1
data_ = s[j + 1:-1]
if data_ != '':
data = literal_eval(data_)
else:
data = []
tmp = s[:j]
break
tmp = tmp.split('.')
# debuginfo(tmp, data)
tmp_ = tmp[:-1]
_tmp = []
for i in tmp_:
a = i.split('[')
b = a[0]
try:
c = literal_eval(a[1][:-1])
except IndexError:
c = None
_tmp.append((b, c))
try:
insert_data = _tmp[-1][1]
_tmp[-1] = _tmp[-1][0], None
if insert_data is not None:
try:
data.insert(0, insert_data)
data = tuple(data)
except AttributeError:
data = tuple([insert_data, data])
else:
data = tuple([data])
except IndexError:
data = tuple(data)
# debuginfo(_tmp, (tmp[-1], data))
return _tmp, (tmp[-1], data)
def clear(self):
self._head = self._tail = None
self._keepDict.clear()
WeakValueDictionary.clear(self)
class JsonCacheHandler(BaseCacheHandler):
"""
This cache handler implements on-disk cache store in the form
of compressed JSON. To improve performance further, it also
keeps loads data from on-disk cache to memory, and uses weakref
object cache for assembled objects.
Required arguments:
cache_path -- file name where on-disk cache will be stored (.json.bz2)
"""
def __init__(self, cache_path):
self._cache_path = os.path.abspath(cache_path)
# Initialize memory data cache
self.__type_data_cache = {}
self.__attribute_data_cache = {}
self.__effect_data_cache = {}
self.__modifier_data_cache = {}
self.__fingerprint = None
# Initialize weakref object cache
self.__type_obj_cache = WeakValueDictionary()
self.__attribute_obj_cache = WeakValueDictionary()
self.__effect_obj_cache = WeakValueDictionary()
self.__modifier_obj_cache = WeakValueDictionary()
# If cache doesn't exist, silently finish initialization
if not os.path.exists(self._cache_path):
return
# Read JSON into local variable
try:
with bz2.BZ2File(self._cache_path, 'r') as file:
json_data = file.read().decode('utf-8')
data = json.loads(json_data)
except KeyboardInterrupt:
raise
# If file doesn't exist, JSON load errors occur, or
# anything else bad happens, do not load anything
# and leave values as initialized
except:
msg = 'error during reading cache'
logger.error(msg)
# Load data into data cache, if no errors occurred
# during JSON reading/parsing
else:
self.__update_mem_cache(data)
def get_type(self, type_id):
try:
type_id = int(type_id)
except TypeError as e:
raise TypeFetchError(type_id) from e
try:
type_ = self.__type_obj_cache[type_id]
except KeyError:
# We do str(int(id)) here because JSON dictionaries
# always have strings as key
json_type_id = str(type_id)
try:
type_data = self.__type_data_cache[json_type_id]
except KeyError as e:
raise TypeFetchError(type_id) from e
type_ = Type(type_id=type_id,
group=type_data[0],
category=type_data[1],
attributes={
attr_id: attr_val
for attr_id, attr_val in type_data[2]
},
effects=tuple(
self.get_effect(effect_id)
for effect_id in type_data[3]),
default_effect=None if type_data[4] is None else
self.get_effect(type_data[4]))
self.__type_obj_cache[type_id] = type_
return type_
def get_attribute(self, attr_id):
try:
attr_id = int(attr_id)
except TypeError as e:
raise AttributeFetchError(attr_id) from e
try:
attribute = self.__attribute_obj_cache[attr_id]
except KeyError:
json_attr_id = str(attr_id)
try:
attr_data = self.__attribute_data_cache[json_attr_id]
except KeyError as e:
raise AttributeFetchError(attr_id) from e
attribute = Attribute(attribute_id=attr_id,
max_attribute=attr_data[0],
default_value=attr_data[1],
high_is_good=attr_data[2],
stackable=attr_data[3])
self.__attribute_obj_cache[attr_id] = attribute
return attribute
def get_effect(self, effect_id):
try:
effect_id = int(effect_id)
except TypeError as e:
raise EffectFetchError(effect_id) from e
try:
effect = self.__effect_obj_cache[effect_id]
except KeyError:
json_effect_id = str(effect_id)
try:
effect_data = self.__effect_data_cache[json_effect_id]
except KeyError as e:
raise EffectFetchError(effect_id) from e
effect = Effect(effect_id=effect_id,
category=effect_data[0],
is_offensive=effect_data[1],
is_assistance=effect_data[2],
duration_attribute=effect_data[3],
discharge_attribute=effect_data[4],
range_attribute=effect_data[5],
falloff_attribute=effect_data[6],
tracking_speed_attribute=effect_data[7],
fitting_usage_chance_attribute=effect_data[8],
build_status=effect_data[9],
modifiers=tuple(
self.get_modifier(modifier_id)
for modifier_id in effect_data[10]))
self.__effect_obj_cache[effect_id] = effect
return effect
def get_modifier(self, modifier_id):
try:
modifier_id = int(modifier_id)
except TypeError as e:
raise ModifierFetchError(modifier_id) from e
try:
modifier = self.__modifier_obj_cache[modifier_id]
except KeyError:
json_modifier_id = str(modifier_id)
try:
modifier_data = self.__modifier_data_cache[json_modifier_id]
except KeyError as e:
raise ModifierFetchError(modifier_id) from e
modifier = Modifier(modifier_id=modifier_id,
state=modifier_data[0],
scope=modifier_data[1],
src_attr=modifier_data[2],
operator=modifier_data[3],
tgt_attr=modifier_data[4],
domain=modifier_data[5],
filter_type=modifier_data[6],
filter_value=modifier_data[7])
self.__modifier_obj_cache[modifier_id] = modifier
return modifier
def get_fingerprint(self):
return self.__fingerprint
def update_cache(self, data, fingerprint):
# Make light version of data and add fingerprint
# to it
data = self.__strip_data(data)
data['fingerprint'] = fingerprint
# Update disk cache
cache_folder = os.path.dirname(self._cache_path)
if os.path.isdir(cache_folder) is not True:
os.makedirs(cache_folder, mode=0o755)
with bz2.BZ2File(self._cache_path, 'w') as file:
json_data = json.dumps(data)
file.write(json_data.encode('utf-8'))
# Update data cache; encode to JSON and decode back
# to make sure form of data is the same as after
# loading it from cache (e.g. dictionary keys are
# stored as strings in JSON)
data = json.loads(json_data)
self.__update_mem_cache(data)
def __strip_data(self, data):
"""
Rework passed data, keying it and stripping dictionary
keys from rows for performance.
"""
slim_data = {}
slim_types = {}
for type_row in data['types']:
type_id = type_row['type_id']
slim_types[type_id] = (
type_row['group'],
type_row['category'],
tuple(type_row['attributes'].items()), # Dictionary -> tuple
tuple(type_row['effects']), # List -> tuple
type_row['default_effect'])
slim_data['types'] = slim_types
slim_attribs = {}
for attr_row in data['attributes']:
attribute_id = attr_row['attribute_id']
slim_attribs[attribute_id] = (attr_row['max_attribute'],
attr_row['default_value'],
attr_row['high_is_good'],
attr_row['stackable'])
slim_data['attributes'] = slim_attribs
slim_effects = {}
for effect_row in data['effects']:
effect_id = effect_row['effect_id']
slim_effects[effect_id] = (
effect_row['effect_category'],
effect_row['is_offensive'],
effect_row['is_assistance'],
effect_row['duration_attribute'],
effect_row['discharge_attribute'],
effect_row['range_attribute'],
effect_row['falloff_attribute'],
effect_row['tracking_speed_attribute'],
effect_row['fitting_usage_chance_attribute'],
effect_row['build_status'],
tuple(effect_row['modifiers']) # List -> tuple
)
slim_data['effects'] = slim_effects
slim_modifiers = {}
for modifier_row in data['modifiers']:
modifier_id = modifier_row['modifier_id']
slim_modifiers[modifier_id] = (modifier_row['state'],
modifier_row['scope'],
modifier_row['src_attr'],
modifier_row['operator'],
modifier_row['tgt_attr'],
modifier_row['domain'],
modifier_row['filter_type'],
modifier_row['filter_value'])
slim_data['modifiers'] = slim_modifiers
return slim_data
def __update_mem_cache(self, data):
"""
Loads data into memory data cache.
Required arguments:
data -- dictionary with data to load
"""
self.__type_data_cache = data['types']
self.__attribute_data_cache = data['attributes']
self.__effect_data_cache = data['effects']
self.__modifier_data_cache = data['modifiers']
self.__fingerprint = data['fingerprint']
# Also clear object cache to make sure objects composed
# from old data are gone
self.__type_obj_cache.clear()
self.__attribute_obj_cache.clear()
self.__effect_obj_cache.clear()
self.__modifier_obj_cache.clear()
def __repr__(self):
spec = [['cache_path', '_cache_path']]
return make_repr_str(self, spec)
class RpcService(object):
""" service for one socket """
UID_LEN = 32
def __init__(self, svr, sock, uid, size=None):
if 0:
self.svr = RpcServer()
self.svr = svr
#self._pool = Pool(size=size)
self.sock = sock
if isinstance(svr, RpcClient):
self.sock_addr = svr.addr
else:
self.sock_addr = self.sock.getpeername()
self.uid = str(uid)
if len(self.uid) != self.UID_LEN:
raise ValueError, 'uid length error: len(uid)=%d <> %d' % (
len(uid), self.UID_LEN)
self._slock = Semaphore()
self._reconnected = None
self.reconnect_timeout = RECONNECT_TIMEOUT
#self.iter_id = itertools.cycle(xrange(MAX_INDEX))
self._next_id = 0
self._resps = {}
self._proxys = WeakValueDictionary()
self.stoped = True
self.sock_error = False
if HEARTBEAT_TIME > 0:
self._heart_time = time.time()
self._heart_task = spawn(self.heartbeat)
self.shells = {}
def next_id(self):
self._next_id += 1
if self._next_id >= MAX_INDEX:
self._next_id = 1
return self._next_id
def start(self):
if not self.stoped:
return
self.stoped = False
self._recv_task = spawn(self._recver)
self._recver_on_error = False
#_services_.append(self.sock_addr)
def remote_stop(self):
#printf('remote_stop:%s', self.sock_addr)
self.sock_error = True
self.stop()
def close(self):
if not self.sock:
return
try:
self.sock._sock.close()
except socket_error:
pass
self.sock.close()
self.sock = None
def stop(self):
if self.stoped:
return
self.stoped = True
self._recv_task.kill(block=0)
self._recv_task = None
if 1 and not self.sock_error:
try:
#printf('remote_stop:%s', self.sock_addr)
self.call('', 'remote_stop', tuple(), None, no_result=True)
sleep(0.01)
except:
pass
self.svr.svc_stop(self)
if getattr(self, '_heart_task', None):
self._heart_task.kill(block=False)
self._heart_task = None
try:
self._stop_resps()
self._stop_proxys()
finally:
self.close()
#_services_.append('-%s' % str(self.sock_addr))
def _stop_resps(self):
error = RpcRuntimeError('service stoped')
for k, v in self._resps.iteritems():
v.set_exception(error)
self._resps.clear()
def _stop_proxys(self):
if not len(self._proxys):
return
proxys = self._proxys.values()
self._proxys.clear()
for p in proxys:
p.on_close()
## def _sender(self):
## running = True
## _send = self.sock.sendall
## try:
## for data in self._send_queue:
## _send('%s%s' %(pack('I', len(data)), data))
## except GreenletExit:
## pass
def _recver(self):
""" 接收处理数据 """
recv_func = self.sock.recv
def _read(c):
d = recv_func(c)
if d:
return d
if self.stoped:
raise GreenletExit
self._recver_on_error = True
self._on_socket_error(None)
self._recver_on_error = False
return None
try:
sio = StringIO()
while not self.stoped:
dlen = 4
d = ''
while dlen > 0:
data = _read(dlen)
if data is None:
continue
d += data
dlen -= len(data)
dlen = unpack('I', d)[0]
#rs = []
sio.seek(0)
sio.truncate()
while dlen > 0:
data = _read(dlen)
if data is None:
continue
#rs.append(data)
sio.write(data)
dlen -= len(data)
#spawn(self._handle, loads(''.join(rs)))
sio.seek(0)
self._handle(load(sio))
#self._pool.spawn(self._handle, loads(''.join(rs)))
except GreenletExit:
pass
except Exception as err:
printf('[RpcService._recver]%s', err)
finally:
self.stop()
def _on_socket_error(self, err):
if self.stoped or self.reconnect_timeout <= 0:
self.sock_error = True
self.stop()
return
def _reconnect():
#尝试重连或等待重连
while not self.stoped:
try:
self.svr.reconnect()
break
except socket_error:
pass
sleep(0.5)
if self._reconnected is None:
self._reconnected = AsyncResult()
printf('socket error:%s, RpcService try reconnect', err)
self.send = self.send_wait
if hasattr(self.svr, 'reconnect'): #RpcClient.reconnect
spawn(_reconnect)
self._wait_reconnect()
def _wait_reconnect(self):
_reconnected = self._reconnected
try:
_reconnected.get(timeout=self.reconnect_timeout)
except Timeout:
pass
if not _reconnected.successful():
self.stop()
if self.sock_error or _reconnected.exception is None:
return
self.sock_error = True
raise _reconnected.exception
def reconnect(self, sock):
## if not self._recver_on_error:
## self._recv_task.kill(exception=socket_error('reconnect'))
self.sock = sock
self.send = self.send_imme
if self._reconnected is not None:
self._reconnected.set(True)
self._reconnected = None
def send_imme(self, *args):
data = dumps(args)
with self._slock:
try:
self.sock.sendall('%s%s' % (pack('I', len(data)), data))
except socket_error as err:
self._on_socket_error(err)
#重新发送
self.sock.sendall('%s%s' % (pack('I', len(data)), data))
## self._send_queue.put(dumps(args))
def send_wait(self, *args):
if self._reconnected is not None:
self._wait_reconnect()
self.send_imme(*args)
send = send_imme
def _read_response(self, index, timeout):
rs = AsyncResult()
self._resps[index] = rs
resp = rs.wait(timeout)
self._resps.pop(index, None)
if not rs.successful():
error = rs.exception
if error is None:
error = Timeout
raise error
return resp
def _reg_obj(self, obj):
if hasattr(obj, 'proxy_pack'):
return obj.proxy_pack(), False
if isinstance(obj, RpcProxy):
return obj._id, False
if hasattr(obj, '_rpc_proxy_'):
return obj._rpc_proxy_(), True
return self.svr.register(obj), False
def call(self,
obj_id,
name,
args,
kw,
no_result=False,
timeout=CALL_TIMEORUT,
pickle=False,
proxy=False):
dtype = RT_REQUEST
if proxy:
objs = args[0] #first arg is proxy(str, RpcProxy or list)
if isinstance(objs, (tuple, list)):
obj_ids = []
for o in objs:
obj, is_pickle = self._reg_obj(o)
pickle = pickle or is_pickle
obj_ids.append(obj)
else:
obj, is_pickle = self._reg_obj(objs)
pickle = pickle or is_pickle
obj_ids = obj
args = (obj_ids, ) + args[1:]
dtype |= DT_PROXY
if pickle:
dtype |= DT_PICKLE
argkw = pickle_dumps((args, kw), PICKLE_PROTOCOL)
else:
argkw = dumps((args, kw))
if len(argkw) >= ZIP_LENGTH:
dtype |= DT_ZIP
argkw = zlib.compress(argkw, ZIP_LEVEL)
if no_result:
dtype |= ST_NO_RESULT
index = self.next_id() #iter_id.next()
self.send(dtype, obj_id, index, name, argkw)
if no_result:
return
result = self._read_response(index, timeout)
return result
def _handle_request(self, parts):
dtype, obj_id, index, name, argkw = parts
try:
obj = self.get_export(obj_id)
if obj is None:
raise RpcExportNoFound, obj_id
func = getattr(obj, name)
if not callable(func):
raise RpcFuncNoFound, name
if dtype & DT_ZIP:
argkw = zlib.decompress(argkw)
if dtype & DT_PICKLE:
args, kw = pickle_loads(argkw)
else:
args, kw = loads(argkw)
if dtype & DT_PROXY:
export_ids = args[0]
if isinstance(export_ids, (tuple, list)):
proxys = []
for e in export_ids:
proxys.append(self.get_proxy(e))
else:
proxys = self.get_proxy(export_ids)
args = (proxys, ) + tuple(args[1:])
if getattr(func, "_block_", True):
spawn(self._handle_request_call, func, args, kw, dtype, index,
obj_id, name, argkw)
else:
self._handle_request_call(func, args, kw, dtype, index, obj_id,
name, argkw)
except Exception as e:
log_except('export(%s).%s(%s)', obj_id, name, repr(argkw))
if dtype & ST_NO_RESULT or self.svr.stoped:
return
self.send(RT_EXCEPTION, index, str(e))
def _handle_request_call(self, func, args, kw, dtype, index, obj_id, name,
argkw):
try:
let = getcurrent()
setattr(let, _service_name_, self)
if args is None:
rs = func()
else:
rs = func(*args, **kw) if kw is not None else func(*args)
if dtype & ST_NO_RESULT:
return
if not getattr(func, '_rpc_pickle_result_', False):
self.send(RT_RESPONSE, index, dumps(rs))
else:
self.send(RT_RESPONSE | DT_PICKLE, index,
pickle_dumps(rs, PICKLE_PROTOCOL))
except Exception as e:
log_except('export(%s).%s(%s)', obj_id, name, repr(argkw))
if dtype & ST_NO_RESULT or self.svr.stoped:
return
self.send(RT_EXCEPTION, index, str(e))
def _handle_response(self, parts):
dtype, index, argkw = parts
try:
rs = self._resps.pop(index)
if dtype & DT_PICKLE:
result = pickle_loads(argkw)
else:
result = loads(argkw)
rs.set(result)
except KeyError:
pass
def _handle_exception(self, parts):
RT_EXCEPTION, index, error = parts
#try:
# error = pickle_loads(error)
#except:
error = RpcCallError(str(error))
try:
rs = self._resps.pop(index)
rs.set_exception(error)
except KeyError:
pass
def _handle(self, parts):
#parts = (parts[0], ) + loads(parts[1]) if len(parts) ==2 else loads(parts[0])
rt = parts[0] & RT_MARK
if rt == RT_REQUEST:
self._handle_request(parts)
elif rt == RT_RESPONSE:
self._handle_response(parts)
elif rt == RT_EXCEPTION:
self._handle_exception(parts)
elif rt == RT_HEARTBEAT:
self._heart_time = time.time()
else:
raise ValueError('unknown data:%s' % str(rt))
def heartbeat(self):
beat = RT_HEARTBEAT
btime = HEARTBEAT_TIME
check_times = HEARTBEAT_TIME * max(3, RECONNECT_TIMEOUT)
try:
while not self.stoped:
self.send(beat)
sleep(btime)
if (self._heart_time + check_times) < time.time():
printf('heartbeat timeout!!!!!!!!')
self.sock_error = True
break
finally:
self.stop()
@classmethod
def handshake_svr(cls, sock):
uid = sock.recv(cls.UID_LEN)
return uid
def handshake_cli(self):
self.sock.sendall(self.uid)
#######remote call##############
def get_export(self, export_id):
""" get export obj by export_name """
if not export_id:
return self
return self.svr.get_export(export_id)
def get_proxy(self, export_id, proxy_cls=None):
""" remote call: get export obj by id """
if isinstance(export_id, RpcProxy):
return export_id
if isinstance(export_id, (tuple, list)):
export_cls = PROXYS[export_id[0]]
return export_cls.proxy_unpack(export_id, svc=self)
if proxy_cls in (None, RpcProxy):
try:
return self._proxys[export_id]
except KeyError:
proxy_cls = RpcProxy
proxy = proxy_cls(export_id, svc=self)
self.reg_proxy(export_id, proxy)
return proxy
else:
p = proxy_cls(export_id, svc=self)
self.reg_proxy(id(p), p)
return p
def reg_proxy(self, key, proxy):
self._proxys[key] = proxy
def stop_shell(self, shell_id):
shell = self.shells.pop(shell_id, None)
if not shell:
return
shell.stop()
def start_shell(self, console_proxy, pre_prompt):
from rpc_shell import RpcShell
if self.svr.access and not self.svr.access.access_shell(self):
printf('[rpc]shell deny:%s', self.sock_addr)
return 0
printf('[rpc]shell start:%s', self.sock_addr)
shell = RpcShell(self, console_proxy, pre_prompt=pre_prompt)
shell.start()
#shell.stop remove shell from self.shells
shell_id = id(shell)
self.shells[shell_id] = shell
return shell_id
def stop_console(self, shell_id):
self.call('',
'stop_shell', (shell_id, ),
None,
no_result=True,
timeout=20)
def start_console(self, pre_prompt='', shell=None):
from rpc_shell import RpcLocalConsole, RpcProxyConsole
console = RpcLocalConsole(self) if shell is None else RpcProxyConsole(
self, shell)
shell_id = self.call('',
'start_shell', (console, pre_prompt),
None,
proxy=True,
timeout=20)
try:
console.wait(shell_id)
finally:
pass
def execute(self, func, args, kw, **others):
return self.svr.execute(func, args, kw)
def valid_proxy(self, export_id):
return self.get_export(export_id) != None
class FilesystemStore(Store):
"""
Save and load objects in a given directory. Uses Python's
standard `pickle` module to serialize objects onto files.
All objects are saved as files in the given directory (default:
`gc3libs.defaults.JOBS_DIR`). The file name is the object ID.
If an object contains references to other `Persistable` objects,
these are saved in the file they would have been saved if the
`save` method was called on them in the first place, and only an
'external reference' is saved in the pickled container. This
ensures that: (1) only one copy of a shared object is ever saved,
and (2) any shared reference to `Persistable` objects is correctly
restored when restoring the container.
The default `idfactory` assigns object IDs by appending a
sequential number to the class name; see class `Id` for
details.
The `protocol` argument specifies the serialization protocol to use,
if different from `gc3libs.persistence.serialization.DEFAULT_PROTOCOL`.
Any extra keyword arguments are ignored for compatibility with
`SqlStore`.
"""
def __init__(self,
directory=gc3libs.defaults.JOBS_DIR,
idfactory=IdFactory(),
protocol=DEFAULT_PROTOCOL,
**extra_args):
if isinstance(directory, Url):
super(FilesystemStore, self).__init__(directory)
directory = directory.path
else:
super(FilesystemStore, self).__init__(
Url(scheme='file', path=os.path.abspath(directory)))
self._directory = directory
self.idfactory = idfactory
self._loaded = WeakValueDictionary()
self._protocol = protocol
@same_docstring_as(Store.invalidate_cache)
def invalidate_cache(self):
self._loaded.clear()
@same_docstring_as(Store.list)
def list(self):
if not os.path.exists(self._directory):
return []
return [
id_ for id_ in os.listdir(self._directory)
if not id_.endswith('.OLD')
]
def _load_from_file(self, path):
"""Auxiliary method for `load`."""
# gc3libs.log.debug("Loading object from file '%s' ...", path)
with open(path, 'rb') as src:
unpickler = make_unpickler(self, src)
obj = unpickler.load()
return obj
@same_docstring_as(Store.load)
def load(self, id_):
# return cached copy, if any
try:
return self._loaded[str(id_)]
except KeyError:
pass
# no cached copy, load from disk
filename = os.path.join(self._directory, id_)
sources = [filename, filename + '.OLD']
for source in sources:
if not os.path.exists(source):
gc3libs.log.debug(
"Cannot load object %s from '%s':"
" file does not exist", id_, source)
continue
try:
obj = self._load_from_file(source)
break # exit `for source in sources` loop ...
except Exception as ex:
gc3libs.log.warning("Failed loading file '%s': %s: %s",
filename,
ex.__class__.__name__,
ex,
exc_info=True)
else:
# complain loudly
raise gc3libs.exceptions.LoadError(
"Failed loading object %s from file(s) %r."
" (Earlier log lines may provide more details.)" %
(id_, sources))
# minimal sanity check
if not hasattr(obj, 'persistent_id'):
raise gc3libs.exceptions.LoadError(
"Invalid format in file '%s':"
" missing 'persistent_id' attribute" % (filename, ))
if str(obj.persistent_id) != str(id_):
raise gc3libs.exceptions.LoadError(
"Retrieved persistent ID '%s' %s"
" does not match given ID '%s' %s" %
(obj.persistent_id, type(obj.persistent_id), id_, type(id_)))
# maybe update object after GC3Pie update?
super(FilesystemStore, self)._update_to_latest_schema()
# update cache
assert str(id_) not in self._loaded
self._loaded[str(id_)] = obj
return obj
@same_docstring_as(Store.remove)
def remove(self, id_):
filename = os.path.join(self._directory, id_)
os.remove(filename)
try:
del self._loaded[str(id_)]
except KeyError:
pass
@same_docstring_as(Store.replace)
def replace(self, id_, obj):
self._save_or_replace(id_, obj)
@same_docstring_as(Store.save)
def save(self, obj):
if not hasattr(obj, 'persistent_id'):
obj.persistent_id = self.idfactory.new(obj)
self._save_or_replace(obj.persistent_id, obj)
return obj.persistent_id
def _save_or_replace(self, id_, obj):
"""
Save `obj` into file identified by `id_`; if no such
destination file exists, create it. Ensure that the
destination file is kept intact in case dumping `obj` fails.
"""
filename = os.path.join(self._directory, id_)
# gc3libs.log.debug("Storing job '%s' into file '%s'", obj, filename)
if not os.path.exists(self._directory):
try:
os.makedirs(self._directory)
except Exception as ex:
# raise same exception but add context message
gc3libs.log.error("Could not create jobs directory '%s': %s" %
(self._directory, str(ex)))
raise
backup = None
if os.path.exists(filename):
backup = filename + '.OLD'
os.rename(filename, backup)
with open(filename, 'w+b') as tgt:
try:
pickler = make_pickler(self, tgt, obj)
pickler.dump(obj)
except Exception as err:
gc3libs.log.error(
"Error saving task '%s' to file '%s': %s: %s", obj,
filename, err.__class__.__name__, err)
# move backup file back in place
if backup is not None:
try:
os.rename(backup, filename)
except:
pass # ignore errors
raise
if hasattr(obj, 'changed'):
obj.changed = False
# remove backup file, if exists
try:
os.remove(backup)
except:
pass # ignore errors
# update cache
if id_ in self._loaded:
old = self._loaded[str(id_)]
if old is not obj:
self._loaded[str(id_)] = obj
class Storage(object):
def __init__(self, cache=None):
# {(InstanceInfo(instance).model_info.model, instance.pk): InstanceInfo, ...}}
self._alive = WeakValueDictionary()
# {InstanceInfo(instance): instance, ...}}
self._dirty = {}
if cache is not None:
self._cache = cache
else:
self._cache = MRUCache(settings.MAX_CACHE)
signals.register(
(
"instance-deleted",
"start-tracking-changes",
"stop-tracking-changes",
"model-pre-init",
"model-post-init",
"model-pre-save",
"model-post-save",
"model-pre-delete",
"model-post-delete",
"model-pre-update",
"model-post-update",
# "model-history-reset",
# "model-history-redo",
# "model-history-undo",
"relation-pre-get",
"relation-post-get",
"relation-pre-set",
"relation-post-set",
"relation-pre-delete",
"relation-post-delete",
"relation-pre-add",
"relation-post-add",
"relation-pre-remove",
"relation-post-remove",
"model-do-cache",
"model-do-not-cache",
),
self
)
signals.register_with_callback("cache-rollback", self, "rollback")
def get(self, query):
return QuerySetIterator(query, self)
def clear(self):
self._dirty.clear()
self._alive.clear()
self._cache.clear()
def set_dirty(self, inst_info):
self._dirty[inst_info] = inst_info.get_inst()
def cache(self, inf):
self._alive.pop((inf.model_info.model, inf._lazypkval), None)
self._alive[(inf.model_info.model, inf.get_pk_as_key())] = inf
self._cache.add(inf)
def uncache(self, inf):
if inf in self._cache:
self._cache.remove(inf)
self._dirty.pop(inf, None)
self._alive.pop((inf.model_info.model, inf.get_pk_as_key()), None)
self._alive.pop((inf.model_info.model, inf._lazypkval), None)
### signals ###
#??? what is this used for?
def instance_deleted(self, inst_info):
print "instance_deleted", inst_info
def model_pre_init(self, instance, **kwargs):
signals.fire("stop-tracking-changes", instance=instance)
def model_post_init(self, instance, **kwargs):
inf = get_inst_info(instance)
signals.fire("start-tracking-changes", instance=instance)
self._alive[(inf.model_info.model, inf.get_pk_as_key())] = inf
# If instance was not initialized with a value for primary key,
# then it has not been saved yet and goes into self._dirty.
if instance.pk is None:
self.set_dirty(inf)
def start_tracking_changes(self, instance):
get_inst_info(instance)._meta["track-changes"] = True
def stop_tracking_changes(self, instance):
get_inst_info(instance)._meta["track-changes"] = False
def model_pre_save(self, instance):
pass
def model_post_save(self, instance, created):
inf = get_inst_info(instance)
if not inf._meta["do-cache"]:
return
self.cache(inf)
self._dirty.pop(inf, None)
# On calling Model.delete, tracking of changes is stopped, so start
# tracking now.
signals.fire("start-tracking-changes", instance=instance)
def model_pre_update(self, instance, value, fieldname):
# instance._inst_info.record_change(fieldname, value)
pass
def model_post_update(self, instance, value, fieldname):
inf = get_inst_info(instance)
if not inf in self._dirty:
self.set_dirty(inf)
def model_pre_delete(self, instance):
signals.fire("stop-tracking-changes", instance=instance)
def model_post_delete(self, instance, deleted):
if not deleted:
return
inf = get_inst_info(instance)
self.uncache(inf)
instance.id = None
def model_do_cache(self, instance):
inf = get_inst_info(instance)
self.cache(inf)
self.set_dirty(inf)
inf._meta["do-cache"] = True
signals.fire("start-tracking-changes", instance=instance)
def model_do_not_cache(self, instance):
inf = get_inst_info(instance)
self.uncache(inf)
inf._meta["do-cache"] = False
signals.fire("stop-tracking-changes", instance=instance)
def model_history_reset(self, instance, **kwargs):
raise NotImplementedError
def model_history_undo(self, instance, fieldname, **kwargs):
raise NotImplementedError
def model_history_redo(self, instance, fieldname, **kwargs):
raise NotImplementedError
def relation_pre_get(self, manager, **kwargs):
pass
def relation_post_get(self, manager, **kwargs):
pass
def relation_pre_set(self, manager, values, **kwargs):
pass
def relation_post_set(self, manager, values, **kwargs):
pass
def relation_pre_delete(self, manager, **kwargs):
pass
def relation_post_delete(self, manager, **kwargs):
pass
def relation_pre_add(self, manager, values, **kwargs):
pass
def relation_post_add(self, manager, values, **kwargs):
pass
def relation_pre_remove(self, manager, values, **kwargs):
pass
def relation_post_remove(self, manager, values, **kwargs):
pass
class JsonCacheHandler(BaseCacheHandler):
"""
This cache handler implements on-disk cache store in the form
of compressed JSON. To improve performance further, it also
keeps loads data from on-disk cache to memory, and uses weakref
object cache for assembled objects.
Required arguments:
cache_path -- file name where on-disk cache will be stored (.json.bz2)
"""
def __init__(self, cache_path):
self._cache_path = os.path.abspath(cache_path)
# Initialize memory data cache
self.__type_data_cache = {}
self.__attribute_data_cache = {}
self.__effect_data_cache = {}
self.__modifier_data_cache = {}
self.__fingerprint = None
# Initialize weakref object cache
self.__type_obj_cache = WeakValueDictionary()
self.__attribute_obj_cache = WeakValueDictionary()
self.__effect_obj_cache = WeakValueDictionary()
self.__modifier_obj_cache = WeakValueDictionary()
# If cache doesn't exist, silently finish initialization
if not os.path.exists(self._cache_path):
return
# Read JSON into local variable
try:
with bz2.BZ2File(self._cache_path, 'r') as file:
json_data = file.read().decode('utf-8')
data = json.loads(json_data)
except KeyboardInterrupt:
raise
# If file doesn't exist, JSON load errors occur, or
# anything else bad happens, do not load anything
# and leave values as initialized
except:
msg = 'error during reading cache'
logger.error(msg)
# Load data into data cache, if no errors occurred
# during JSON reading/parsing
else:
self.__update_mem_cache(data)
def get_type(self, type_id):
try:
type_id = int(type_id)
except TypeError as e:
raise TypeFetchError(type_id) from e
try:
type_ = self.__type_obj_cache[type_id]
except KeyError:
# We do str(int(id)) here because JSON dictionaries
# always have strings as key
json_type_id = str(type_id)
try:
type_data = self.__type_data_cache[json_type_id]
except KeyError as e:
raise TypeFetchError(type_id) from e
type_ = Type(
type_id=type_id,
group=type_data[0],
category=type_data[1],
attributes={attr_id: attr_val for attr_id, attr_val in type_data[2]},
effects=tuple(self.get_effect(effect_id) for effect_id in type_data[3]),
default_effect=None if type_data[4] is None else self.get_effect(type_data[4])
)
self.__type_obj_cache[type_id] = type_
return type_
def get_attribute(self, attr_id):
try:
attr_id = int(attr_id)
except TypeError as e:
raise AttributeFetchError(attr_id) from e
try:
attribute = self.__attribute_obj_cache[attr_id]
except KeyError:
json_attr_id = str(attr_id)
try:
attr_data = self.__attribute_data_cache[json_attr_id]
except KeyError as e:
raise AttributeFetchError(attr_id) from e
attribute = Attribute(
attribute_id=attr_id,
max_attribute=attr_data[0],
default_value=attr_data[1],
high_is_good=attr_data[2],
stackable=attr_data[3]
)
self.__attribute_obj_cache[attr_id] = attribute
return attribute
def get_effect(self, effect_id):
try:
effect_id = int(effect_id)
except TypeError as e:
raise EffectFetchError(effect_id) from e
try:
effect = self.__effect_obj_cache[effect_id]
except KeyError:
json_effect_id = str(effect_id)
try:
effect_data = self.__effect_data_cache[json_effect_id]
except KeyError as e:
raise EffectFetchError(effect_id) from e
effect = Effect(
effect_id=effect_id,
category=effect_data[0],
is_offensive=effect_data[1],
is_assistance=effect_data[2],
duration_attribute=effect_data[3],
discharge_attribute=effect_data[4],
range_attribute=effect_data[5],
falloff_attribute=effect_data[6],
tracking_speed_attribute=effect_data[7],
fitting_usage_chance_attribute=effect_data[8],
build_status=effect_data[9],
modifiers=tuple(self.get_modifier(modifier_id) for modifier_id in effect_data[10])
)
self.__effect_obj_cache[effect_id] = effect
return effect
def get_modifier(self, modifier_id):
try:
modifier_id = int(modifier_id)
except TypeError as e:
raise ModifierFetchError(modifier_id) from e
try:
modifier = self.__modifier_obj_cache[modifier_id]
except KeyError:
json_modifier_id = str(modifier_id)
try:
modifier_data = self.__modifier_data_cache[json_modifier_id]
except KeyError as e:
raise ModifierFetchError(modifier_id) from e
modifier = Modifier(
modifier_id=modifier_id,
state=modifier_data[0],
scope=modifier_data[1],
src_attr=modifier_data[2],
operator=modifier_data[3],
tgt_attr=modifier_data[4],
domain=modifier_data[5],
filter_type=modifier_data[6],
filter_value=modifier_data[7]
)
self.__modifier_obj_cache[modifier_id] = modifier
return modifier
def get_fingerprint(self):
return self.__fingerprint
def update_cache(self, data, fingerprint):
# Make light version of data and add fingerprint
# to it
data = self.__strip_data(data)
data['fingerprint'] = fingerprint
# Update disk cache
cache_folder = os.path.dirname(self._cache_path)
if os.path.isdir(cache_folder) is not True:
os.makedirs(cache_folder, mode=0o755)
with bz2.BZ2File(self._cache_path, 'w') as file:
json_data = json.dumps(data)
file.write(json_data.encode('utf-8'))
# Update data cache; encode to JSON and decode back
# to make sure form of data is the same as after
# loading it from cache (e.g. dictionary keys are
# stored as strings in JSON)
data = json.loads(json_data)
self.__update_mem_cache(data)
def __strip_data(self, data):
"""
Rework passed data, keying it and stripping dictionary
keys from rows for performance.
"""
slim_data = {}
slim_types = {}
for type_row in data['types']:
type_id = type_row['type_id']
slim_types[type_id] = (
type_row['group'],
type_row['category'],
tuple(type_row['attributes'].items()), # Dictionary -> tuple
tuple(type_row['effects']), # List -> tuple
type_row['default_effect']
)
slim_data['types'] = slim_types
slim_attribs = {}
for attr_row in data['attributes']:
attribute_id = attr_row['attribute_id']
slim_attribs[attribute_id] = (
attr_row['max_attribute'],
attr_row['default_value'],
attr_row['high_is_good'],
attr_row['stackable']
)
slim_data['attributes'] = slim_attribs
slim_effects = {}
for effect_row in data['effects']:
effect_id = effect_row['effect_id']
slim_effects[effect_id] = (
effect_row['effect_category'],
effect_row['is_offensive'],
effect_row['is_assistance'],
effect_row['duration_attribute'],
effect_row['discharge_attribute'],
effect_row['range_attribute'],
effect_row['falloff_attribute'],
effect_row['tracking_speed_attribute'],
effect_row['fitting_usage_chance_attribute'],
effect_row['build_status'],
tuple(effect_row['modifiers']) # List -> tuple
)
slim_data['effects'] = slim_effects
slim_modifiers = {}
for modifier_row in data['modifiers']:
modifier_id = modifier_row['modifier_id']
slim_modifiers[modifier_id] = (
modifier_row['state'],
modifier_row['scope'],
modifier_row['src_attr'],
modifier_row['operator'],
modifier_row['tgt_attr'],
modifier_row['domain'],
modifier_row['filter_type'],
modifier_row['filter_value']
)
slim_data['modifiers'] = slim_modifiers
return slim_data
def __update_mem_cache(self, data):
"""
Loads data into memory data cache.
Required arguments:
data -- dictionary with data to load
"""
self.__type_data_cache = data['types']
self.__attribute_data_cache = data['attributes']
self.__effect_data_cache = data['effects']
self.__modifier_data_cache = data['modifiers']
self.__fingerprint = data['fingerprint']
# Also clear object cache to make sure objects composed
# from old data are gone
self.__type_obj_cache.clear()
self.__attribute_obj_cache.clear()
self.__effect_obj_cache.clear()
self.__modifier_obj_cache.clear()
def __repr__(self):
spec = [['cache_path', '_cache_path']]
return make_repr_str(self, spec)
class JsonCacheHandler(CacheHandler):
"""
This cache handler implements on-disk cache store in the form
of compressed JSON. To improve performance further, it also
keeps loads data from on-disk cache to memory, and uses weakref
object cache for assembled objects.
Positional arguments:
diskCacheFolder -- folder where on-disk cache files are stored
name -- unique indentifier of cache, e.g. Eos instance name
logger -- logger to use for errors
"""
def __init__(self, diskCacheFolder, name, logger):
self._diskCacheFile = os.path.join(diskCacheFolder,
'{}.json.bz2'.format(name))
self._logger = logger
# Initialize memory data cache
self.__typeDataCache = {}
self.__attributeDataCache = {}
self.__effectDataCache = {}
self.__modifierDataCache = {}
self.__fingerprint = None
# Initialize weakref object cache
self.__typeObjCache = WeakValueDictionary()
self.__attributeObjCache = WeakValueDictionary()
self.__effectObjCache = WeakValueDictionary()
self.__modifierObjCache = WeakValueDictionary()
# If cache doesn't exist, silently finish initialization
if not os.path.exists(self._diskCacheFile):
return
# Read JSON into local variable
try:
with bz2.BZ2File(self._diskCacheFile, 'r') as file:
jsonData = file.read().decode('utf-8')
data = json.loads(jsonData)
except KeyboardInterrupt:
raise
# If file doesn't exist, JSON load errors occur, or
# anything else bad happens, do not load anything
# and leave values as initialized
except:
msg = 'error during reading cache'
self._logger.error(msg, childName='cacheHandler')
# Load data into data cache, if no errors occurred
# during JSON reading/parsing
else:
self.__updateMemCache(data)
def getType(self, typeId):
try:
type_ = self.__typeObjCache[typeId]
except KeyError:
# We do str(int(id)) here because JSON dictionaries
# always have strings as key
jsonTypeId = str(int(typeId))
try:
data = self.__typeDataCache[jsonTypeId]
except KeyError as e:
raise TypeFetchError(typeId) from e
groupId, catId, duration, discharge, optimal, falloff, tracking, fittable, effects, attribs = data
type_ = Type(
typeId=typeId,
groupId=groupId,
categoryId=catId,
durationAttributeId=duration,
dischargeAttributeId=discharge,
rangeAttributeId=optimal,
falloffAttributeId=falloff,
trackingSpeedAttributeId=tracking,
fittableNonSingleton=fittable,
attributes={attrId: attrVal
for attrId, attrVal in attribs},
effects=tuple(
self.getEffect(effectId) for effectId in effects))
self.__typeObjCache[typeId] = type_
return type_
def getAttribute(self, attrId):
try:
attribute = self.__attributeObjCache[attrId]
except KeyError:
jsonAttrId = str(int(attrId))
try:
data = self.__attributeDataCache[jsonAttrId]
except KeyError as e:
raise AttributeFetchError(attrId) from e
maxAttributeId, defaultValue, highIsGood, stackable = data
attribute = Attribute(attributeId=attrId,
maxAttributeId=maxAttributeId,
defaultValue=defaultValue,
highIsGood=highIsGood,
stackable=stackable)
self.__attributeObjCache[attrId] = attribute
return attribute
def getEffect(self, effectId):
try:
effect = self.__effectObjCache[effectId]
except KeyError:
jsonEffectId = str(int(effectId))
try:
data = self.__effectDataCache[jsonEffectId]
except KeyError as e:
raise EffectFetchError(effectId) from e
effCategoryId, isOffence, isAssist, fitChanceId, buildStatus, modifiers = data
effect = Effect(effectId=effectId,
categoryId=effCategoryId,
isOffensive=isOffence,
isAssistance=isAssist,
fittingUsageChanceAttributeId=fitChanceId,
buildStatus=buildStatus,
modifiers=tuple(
self.getModifier(modifierId)
for modifierId in modifiers))
self.__effectObjCache[effectId] = effect
return effect
def getModifier(self, modifierId):
try:
modifier = self.__modifierObjCache[modifierId]
except KeyError:
jsonModifierId = str(int(modifierId))
try:
data = self.__modifierDataCache[jsonModifierId]
except KeyError as e:
raise ModifierFetchError(modifierId) from e
state, context, srcAttrId, operator, tgtAttrId, location, filType, filValue = data
modifier = Modifier(modifierId=modifierId,
state=state,
context=context,
sourceAttributeId=srcAttrId,
operator=operator,
targetAttributeId=tgtAttrId,
location=location,
filterType=filType,
filterValue=filValue)
self.__modifierObjCache[modifierId] = modifier
return modifier
def getFingerprint(self):
return self.__fingerprint
def updateCache(self, data, fingerprint):
# Make light version of data and add fingerprint
# to it
data = self.__stripData(data)
data['fingerprint'] = fingerprint
# Update disk cache
os.makedirs(os.path.dirname(self._diskCacheFile),
mode=0o755,
exist_ok=True)
with bz2.BZ2File(self._diskCacheFile, 'w') as file:
jsonData = json.dumps(data)
file.write(jsonData.encode('utf-8'))
# Update data cache; encode to JSON and decode back
# to make sure form of data is the same as after
# loading it from cache (e.g. dictionary keys are
# stored as strings in JSON)
data = json.loads(jsonData)
self.__updateMemCache(data)
def __stripData(self, data):
"""
Rework passed data, keying it and stripping dictionary
keys from rows for performance.
"""
slimData = {}
slimTypes = {}
for typeRow in data['types']:
typeId = typeRow['typeId']
slimTypes[typeId] = (
typeRow['groupId'],
typeRow['categoryId'],
typeRow['durationAttributeId'],
typeRow['dischargeAttributeId'],
typeRow['rangeAttributeId'],
typeRow['falloffAttributeId'],
typeRow['trackingSpeedAttributeId'],
typeRow['fittableNonSingleton'],
tuple(typeRow['effects']), # List -> tuple
tuple(typeRow['attributes'].items())) # Dictionary -> tuple
slimData['types'] = slimTypes
slimAttribs = {}
for attrRow in data['attributes']:
attrId = attrRow['attributeId']
slimAttribs[attrId] = (attrRow['maxAttributeId'],
attrRow['defaultValue'],
attrRow['highIsGood'], attrRow['stackable'])
slimData['attributes'] = slimAttribs
slimEffects = {}
for effectRow in data['effects']:
effectId = effectRow['effectId']
slimEffects[effectId] = (
effectRow['effectCategory'], effectRow['isOffensive'],
effectRow['isAssistance'],
effectRow['fittingUsageChanceAttributeId'],
effectRow['buildStatus'], tuple(effectRow['modifiers'])
) # List -> tuple
slimData['effects'] = slimEffects
slimModifiers = {}
for modifierRow in data['modifiers']:
modifierId = modifierRow['modifierId']
slimModifiers[modifierId] = (modifierRow['state'],
modifierRow['context'],
modifierRow['sourceAttributeId'],
modifierRow['operator'],
modifierRow['targetAttributeId'],
modifierRow['location'],
modifierRow['filterType'],
modifierRow['filterValue'])
slimData['modifiers'] = slimModifiers
return slimData
def __updateMemCache(self, data):
"""
Loads data into memory data cache.
Positional arguments:
data -- dictionary with data to load
"""
self.__typeDataCache = data['types']
self.__attributeDataCache = data['attributes']
self.__effectDataCache = data['effects']
self.__modifierDataCache = data['modifiers']
self.__fingerprint = data['fingerprint']
# Also clear object cache to make sure objects composed
# from old data are gone
self.__typeObjCache.clear()
self.__attributeObjCache.clear()
self.__effectObjCache.clear()
self.__modifierObjCache.clear()
class Signal(object):
def __init__(self, *args):
self.__slots = WeakValueDictionary()
for slot in args:
self.connect(slot)
def __call__(self, slot, *args, **kwargs):
"""
Emit signal. If slot passed signal will be called only for this
slot, for all connected slots otherwise.
Calling this method directly lead to immediate signal processing.
It may be not thread-safe. Use emit method from this module for
delayed calling of signals.
"""
if slot is not None:
slots = (self.__slots[self.key(slot)],)
else:
slots = self.__slots.values()
for func in slots:
func(*args, **kwargs)
def key(self, slot):
"""
Get local key name for slot.
"""
if type(slot) == types.FunctionType:
key = (slot.__module__, slot.__name__)
elif type(slot) == types.MethodType:
key = (slot.__func__, id(slot.__self__))
elif isinstance(slot, basestring):
if not slot in registred_slots.keys():
raise ValueError('Slot {0} does not exists.'.format(slot))
key = slot
else:
raise ValueError('Slot {0} has non-slot type'.format(slot))
return key
def connect(self, slot):
"""
Connect signal to slot. Slot may be function, instance method
or name of function perviously registred by `slot` decorator.
"""
key = self.key(slot)
if type(slot) == types.FunctionType:
self.__slots[key] = slot
elif type(slot) == types.MethodType:
self.__slots[key] = partial(slot.__func__, slot.__self__)
elif isinstance(slot, basestring):
self.__slots[key] = registred_slots[slot]
def disconnect(self, slot):
"""
Remove slot from signal connetions.
"""
key = self.key(slot)
del self.__slots[key]
def clear(self):
"""
Disconnect all slots from signal.
"""
self.__slots.clear()
def clear(self):
self._keepDict.clear()
WeakValueDictionary.clear(self)
class DispatchTree(object):
def _display_(self):
'''
Debug purpose method, returns a basic display of the dispatch tree as html
'''
startTimer = time.time()
timeout = 2.0
result="<html><head><style>table,th,td { margin: 5px; border-collapse:collapse; border:1px solid black; }</style></head><body font-family='verdana'>"
result +="<h3>Pools: %r</h3><table>" % len(self.pools)
for i,curr in enumerate(self.pools):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.pools[curr])
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Rendernodes: %r</h3><table>" % len(self.renderNodes)
for i,curr in enumerate(self.renderNodes):
result += "<tr><td>%r</td><td>%r</td></tr>" % (i, self.renderNodes[curr])
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>PoolShares: (attribution de parc pour une tache fille du root, on attribue pas de poolshare aux autres)</h3><table>"
for i,curr in enumerate(self.poolShares):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.poolShares[curr])
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Main level nodes (proxy info only):</h3><table>"
result +="<tr><th>id</th><th>name</th><th>readyCommandCount</th><th>commandCount</th><th>completion</th><th>poolshares</th></tr>"
for i,curr in enumerate(self.nodes[1].children):
result += "<tr><td>%r</td><td>%s</td><td>%d</td><td>%d</td><td>%.2f</td><td>%s</td></tr>" % (i, curr.name, curr.readyCommandCount, curr.commandCount, curr.completion, curr.poolShares.values())
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>All nodes:</h3><table>"
for i,curr in enumerate(self.nodes):
result += "<tr><td>%d</td><td>%s</td><td>%r</td></tr>" % (i, curr, self.nodes[curr].name)
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Tasks:</h3><table>"
for i,curr in enumerate(self.tasks):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, repr(self.tasks[curr]) )
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Commands:</h3><table>"
for i,curr in enumerate(self.commands):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, self.commands[curr] )
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="<h3>Rules:</h3><table>"
for i,curr in enumerate(self.rules):
result += "<tr><td>%r</td><td>%s</td></tr>" % (i, curr )
if (time.time()-startTimer) > timeout:
raise TimeoutException("TimeoutException occured: the dispatchTree might be too large to dump")
result+="</table>"
result +="</body></html>"
logger.info("DispatchTree printed in %.6f s" % (time.time()-startTimer) )
return result
def __init__(self):
# core data
self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy())
self.nodes = WeakValueDictionary()
self.nodes[0] = self.root
self.pools = {}
self.renderNodes = {}
self.tasks = {}
self.rules = []
self.poolShares = {}
self.commands = {}
# deduced properties
self.nodeMaxId = 0
self.poolMaxId = 0
self.renderNodeMaxId = 0
self.taskMaxId = 0
self.commandMaxId = 0
self.poolShareMaxId = 0
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
# listeners
self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange)
self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange)
# # JSA
# self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange)
self.renderNodeListener = ObjectListener(self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange)
self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange)
self.commandListener = ObjectListener(onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange)
self.poolShareListener = ObjectListener(self.onPoolShareCreation)
self.modifiedNodes = []
def registerModelListeners(self):
BaseNode.changeListeners.append(self.nodeListener)
Task.changeListeners.append(self.taskListener)
TaskGroup.changeListeners.append(self.taskListener)
RenderNode.changeListeners.append(self.renderNodeListener)
Pool.changeListeners.append(self.poolListener)
Command.changeListeners.append(self.commandListener)
PoolShare.changeListeners.append(self.poolShareListener)
def destroy(self):
BaseNode.changeListeners.remove(self.nodeListener)
Task.changeListeners.remove(self.taskListener)
RenderNode.changeListeners.remove(self.renderNodeListener)
Pool.changeListeners.remove(self.poolListener)
Command.changeListeners.remove(self.commandListener)
PoolShare.changeListeners.remove(self.poolShareListener)
self.root = None
self.nodes.clear()
self.pools.clear()
self.renderNodes.clear()
self.tasks.clear()
self.rules = None
self.commands.clear()
self.poolShares = None
self.modifiedNodes = None
self.toCreateElements = None
self.toModifyElements = None
self.toArchiveElements = None
def findNodeByPath(self, path, default=None):
nodenames = splitpath(path)
node = self.root
for name in nodenames:
for child in node.children:
if child.name == name:
node = child
break
else:
return default
return node
def updateCompletionAndStatus(self):
self.root.updateCompletionAndStatus()
def validateDependencies(self):
nodes = set()
for dependency in self.modifiedNodes:
for node in dependency.reverseDependencies:
nodes.add(node)
del self.modifiedNodes[:]
for node in nodes:
# logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) )
if not hasattr(node,"task"):
continue
if isinstance(node, TaskNode):
if node.checkDependenciesSatisfaction():
for cmd in node.task.commands:
if cmd.status == CMD_BLOCKED:
cmd.status = CMD_READY
else:
for cmd in node.task.commands:
if cmd.status == CMD_READY:
cmd.status = CMD_BLOCKED
# TODO: may be needed to check dependencies on task groups
# so far, a hack is done on the client side when submitting:
# dependencies of a taksgroup are reported on each task of its heirarchy
#
# elif isinstance(node, FolderNode):
#
# if node.checkDependenciesSatisfaction():
# for cmd in node.getAllCommands():
# if cmd.status == CMD_BLOCKED:
# cmd.status = CMD_READY
# else:
# for cmd in node.getAllCommands():
# if cmd.status == CMD_READY:
# cmd.status = CMD_BLOCKED
def registerNewGraph(self, graph):
user = graph['user']
taskDefs = graph['tasks']
poolName = graph['poolName']
if 'maxRN' in graph.items():
maxRN = int(graph['maxRN'])
else:
maxRN = -1
#
# Create objects.
#
tasks = [None for i in xrange(len(taskDefs))]
for (index, taskDef) in enumerate(taskDefs):
if taskDef['type'] == 'Task':
task = self._createTaskFromJSON(taskDef, user)
elif taskDef['type'] == 'TaskGroup':
task = self._createTaskGroupFromJSON(taskDef, user)
tasks[index] = task
root = tasks[graph['root']]
# get the pool
try:
pool = self.pools[poolName]
except KeyError:
pool = Pool(None, poolName)
self.pools[poolName] = pool
#
# Rebuild full job hierarchy
#
for (taskDef, task) in zip(taskDefs, tasks):
if taskDef['type'] == 'TaskGroup':
for taskIndex in taskDef['tasks']:
task.addTask(tasks[taskIndex])
tasks[taskIndex].parent = task
#
# Compute dependencies for each created task or taskgroup object.
#
dependencies = {}
for (taskDef, task) in zip(taskDefs, tasks):
taskDependencies = {}
if not isinstance(taskDef['dependencies'], list):
raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies'])
if not all(((isinstance(i, int) and
isinstance(sl, list) and
all((isinstance(s, int) for s in sl))) for (i, sl) in taskDef['dependencies'])):
raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies'])
for (taskIndex, statusList) in taskDef['dependencies']:
taskDependencies[tasks[taskIndex]] = statusList
dependencies[task] = taskDependencies
#
# Apply rules to generate dispatch tree nodes.
#
if not self.rules:
logger.warning("graph submitted but no rule has been defined")
unprocessedTasks = [root]
nodes = []
while unprocessedTasks:
unprocessedTask = unprocessedTasks.pop(0)
for rule in self.rules:
try:
nodes += rule.apply(unprocessedTask)
except RuleError:
logger.warning("rule %s failed for graph %s" % (rule, graph))
raise
if isinstance(unprocessedTask, TaskGroup):
for task in unprocessedTask:
unprocessedTasks.append(task)
# create the poolshare, if any, and affect it to the node
if pool:
# FIXME nodes[0] may not be the root node of the graph...
ps = PoolShare(None, pool, nodes[0], maxRN)
# if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true
if maxRN != -1:
ps.userDefinedMaxRN = True
#
# Process dependencies
#
for rule in self.rules:
rule.processDependencies(dependencies)
for node in nodes:
assert isinstance(node.id, int)
self.nodes[node.id] = node
# Init number of command in hierarchy
self.populateCommandCounts(nodes[0])
return nodes
def populateCommandCounts(self, node):
"""
Updates "commandCount" over a whole hierarchy starting from the given node.
"""
res = 0
if isinstance(node, FolderNode):
for child in node.children:
res += self.populateCommandCounts( child )
elif isinstance(node, TaskNode):
res = len(node.task.commands)
node.commandCount = res
return res
def _createTaskGroupFromJSON(self, taskGroupDefinition, user):
# name, parent, arguments, environment, priority, dispatchKey, strategy
id = None
name = taskGroupDefinition['name']
parent = None
arguments = taskGroupDefinition['arguments']
environment = taskGroupDefinition['environment']
requirements = taskGroupDefinition['requirements']
maxRN = taskGroupDefinition['maxRN']
priority = taskGroupDefinition['priority']
dispatchKey = taskGroupDefinition['dispatchKey']
strategy = taskGroupDefinition['strategy']
strategy = loadStrategyClass(strategy.encode())
strategy = strategy()
tags = taskGroupDefinition['tags']
timer = None
if 'timer' in taskGroupDefinition.keys():
timer = taskGroupDefinition['timer']
return TaskGroup(id, name, parent, user, arguments, environment, requirements,
maxRN, priority, dispatchKey, strategy, tags=tags, timer=timer)
def _createTaskFromJSON(self, taskDefinition, user):
# id, name, parent, user, priority, dispatchKey, runner, arguments,
# validationExpression, commands, requirements=[], minNbCores=1,
# maxNbCores=0, ramUse=0, environment={}
name = taskDefinition['name']
runner = taskDefinition['runner']
arguments = taskDefinition['arguments']
environment = taskDefinition['environment']
requirements = taskDefinition['requirements']
maxRN = taskDefinition['maxRN']
priority = taskDefinition['priority']
dispatchKey = taskDefinition['dispatchKey']
validationExpression = taskDefinition['validationExpression']
minNbCores = taskDefinition['minNbCores']
maxNbCores = taskDefinition['maxNbCores']
ramUse = taskDefinition['ramUse']
lic = taskDefinition['lic']
tags = taskDefinition['tags']
timer = None
if 'timer' in taskDefinition.keys():
timer = taskDefinition['timer']
task = Task(None, name, None, user, maxRN, priority, dispatchKey, runner,
arguments, validationExpression, [], requirements, minNbCores,
maxNbCores, ramUse, environment, lic=lic, tags=tags, timer=timer)
for commandDef in taskDefinition['commands']:
description = commandDef['description']
arguments = commandDef['arguments']
cmd = Command(None, description, task, arguments)
task.commands.append(cmd)
# import sys
# logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd)))
return task
## Resets the lists of elements to create or update in the database.
#
def resetDbElements(self):
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
## Recalculates the max ids of all elements. Generally called after a reload from db.
#
def recomputeMaxIds(self):
self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0
self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0
self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0
self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0
self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0
self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0
## Removes from the dispatchtree the provided element and all its parents and children.
#
def unregisterElementsFromTree(self, element):
# /////////////// Handling of the Task
if isinstance(element, Task):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for cmd in element.commands:
self.unregisterElementsFromTree(cmd)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskGroup
elif isinstance(element, TaskGroup):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for task in element.tasks:
self.unregisterElementsFromTree(task)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskNode
elif isinstance(element, TaskNode):
# remove the element from the children of the parent
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the FolderNode
elif isinstance(element, FolderNode):
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the Command
elif isinstance(element, Command):
del self.commands[element.id]
self.toArchiveElements.append(element)
### methods called after interaction with a Task
def onTaskCreation(self, task):
# logger.info(" -- on task creation: %s" % task)
if task.id == None:
self.taskMaxId += 1
task.id = self.taskMaxId
self.toCreateElements.append(task)
else:
self.taskMaxId = max(self.taskMaxId, task.id)
self.tasks[task.id] = task
def onTaskDestruction(self, task):
# logger.info(" -- on task destruction: %s" % task)
self.unregisterElementsFromTree(task)
def onTaskChange(self, task, field, oldvalue, newvalue):
"""
Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes
However in order to keep track of comments (stored in task's tags[comment] field), we make the following change:
- enable task/taskgroups update in DB (cf pulidb.py)
- disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs
BUT the only field we want to update: "tags"
"""
if field == "tags":
self.toModifyElements.append(task)
### methods called after interaction with a BaseNode
def onNodeCreation(self, node):
# logger.info(" -- on node creation: %s" % node)
if node.id == None:
self.nodeMaxId += 1
node.id = self.nodeMaxId
self.toCreateElements.append(node)
else:
self.nodeMaxId = max(self.nodeMaxId, node.id)
if node.parent == None:
node.parent = self.root
def onNodeDestruction(self, node):
# logger.info(" -- on node destruction: %s" % node)
del self.nodes[node.id]
def onNodeChange(self, node, field, oldvalue, newvalue):
# logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) )
# FIXME: do something when nodes are reparented from or to the root node
if node.id is not None:
self.toModifyElements.append(node)
if field == "status" and node.reverseDependencies:
self.modifiedNodes.append(node)
### methods called after interaction with a RenderNode
def onRenderNodeCreation(self, renderNode):
if renderNode.id == None:
self.renderNodeMaxId += 1
renderNode.id = self.renderNodeMaxId
self.toCreateElements.append(renderNode)
else:
self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id)
self.renderNodes[renderNode.name] = renderNode
def onRenderNodeDestruction(self, rendernode):
try:
del self.renderNodes[rendernode.name]
self.toArchiveElements.append(rendernode)
except KeyError, e:
# TOFIX: use of class method vs obj method in changeListener might generate a duplicate call
logger.warning("RN %s seems to have been deleted already." % rendernode.name)
class DispatchTree(object):
def __init__(self):
# core data
self.root = FolderNode(0, "root", None, "root", 1, 1, 0, FifoStrategy())
self.nodes = WeakValueDictionary()
self.nodes[0] = self.root
self.pools = {}
self.renderNodes = {}
self.tasks = {}
self.rules = []
self.poolShares = {}
self.commands = {}
# deduced properties
self.nodeMaxId = 0
self.poolMaxId = 0
self.renderNodeMaxId = 0
self.taskMaxId = 0
self.commandMaxId = 0
self.poolShareMaxId = 0
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
# listeners
self.nodeListener = ObjectListener(self.onNodeCreation, self.onNodeDestruction, self.onNodeChange)
self.taskListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskChange)
self.renderNodeListener = ObjectListener(self.onRenderNodeCreation, self.onRenderNodeDestruction, self.onRenderNodeChange)
self.poolListener = ObjectListener(self.onPoolCreation, self.onPoolDestruction, self.onPoolChange)
self.commandListener = ObjectListener(onCreationEvent=self.onCommandCreation, onChangeEvent=self.onCommandChange)
self.poolShareListener = ObjectListener(self.onPoolShareCreation)
self.modifiedNodes = []
def registerModelListeners(self):
BaseNode.changeListeners.append(self.nodeListener)
Task.changeListeners.append(self.taskListener)
TaskGroup.changeListeners.append(self.taskListener)
RenderNode.changeListeners.append(self.renderNodeListener)
Pool.changeListeners.append(self.poolListener)
Command.changeListeners.append(self.commandListener)
PoolShare.changeListeners.append(self.poolShareListener)
def destroy(self):
BaseNode.changeListeners.remove(self.nodeListener)
Task.changeListeners.remove(self.taskListener)
RenderNode.changeListeners.remove(self.renderNodeListener)
Pool.changeListeners.remove(self.poolListener)
Command.changeListeners.remove(self.commandListener)
PoolShare.changeListeners.remove(self.poolShareListener)
self.root = None
self.nodes.clear()
self.pools.clear()
self.renderNodes.clear()
self.tasks.clear()
self.rules = None
self.commands.clear()
self.poolShares = None
self.modifiedNodes = None
self.toCreateElements = None
self.toModifyElements = None
self.toArchiveElements = None
def findNodeByPath(self, path, default=None):
nodenames = splitpath(path)
node = self.root
for name in nodenames:
for child in node.children:
if child.name == name:
node = child
break
else:
return default
return node
def updateCompletionAndStatus(self):
self.root.updateCompletionAndStatus()
def validateDependencies(self):
nodes = set()
for dependency in self.modifiedNodes:
for node in dependency.reverseDependencies:
nodes.add(node)
del self.modifiedNodes[:]
for node in nodes:
if isinstance(node, TaskNode):
if node.checkDependenciesSatisfaction():
for cmd in node.task.commands:
if cmd.status == CMD_BLOCKED:
cmd.status = CMD_READY
else:
for cmd in node.task.commands:
if cmd.status == CMD_READY:
cmd.status = CMD_BLOCKED
def registerNewGraph(self, graph):
user = graph['user']
taskDefs = graph['tasks']
poolName = graph['poolName']
if 'maxRN' in graph.items():
maxRN = int(graph['maxRN'])
else:
maxRN = -1
#
# Create objects.
#
tasks = [None for i in xrange(len(taskDefs))]
for (index, taskDef) in enumerate(taskDefs):
if taskDef['type'] == 'Task':
task = self._createTaskFromJSON(taskDef, user)
elif taskDef['type'] == 'TaskGroup':
task = self._createTaskGroupFromJSON(taskDef, user)
tasks[index] = task
root = tasks[graph['root']]
# get the pool
try:
pool = self.pools[poolName]
except KeyError:
pool = Pool(None, poolName)
self.pools[poolName] = pool
#
# Rebuild full job hierarchy
#
for (taskDef, task) in zip(taskDefs, tasks):
if taskDef['type'] == 'TaskGroup':
for taskIndex in taskDef['tasks']:
task.addTask(tasks[taskIndex])
tasks[taskIndex].parent = task
#
# Compute dependencies for each created task or taskgroup object.
#
dependencies = {}
for (taskDef, task) in zip(taskDefs, tasks):
taskDependencies = {}
if not isinstance(taskDef['dependencies'], list):
raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies'])
if not all(((isinstance(i, int) and
isinstance(sl, list) and
all((isinstance(s, int) for s in sl))) for (i, sl) in taskDef['dependencies'])):
raise SyntaxError("Dependencies must be a list of (taskId, [status-list]), got %r." % taskDef['dependencies'])
for (taskIndex, statusList) in taskDef['dependencies']:
taskDependencies[tasks[taskIndex]] = statusList
dependencies[task] = taskDependencies
#
# Apply rules to generate dispatch tree nodes.
#
if not self.rules:
logger.warning("graph submitted but no rule has been defined")
unprocessedTasks = [root]
nodes = []
while unprocessedTasks:
unprocessedTask = unprocessedTasks.pop(0)
for rule in self.rules:
try:
nodes += rule.apply(unprocessedTask)
except RuleError:
logger.warning("rule %s failed for graph %s" % (rule, graph))
raise
if isinstance(unprocessedTask, TaskGroup):
for task in unprocessedTask:
unprocessedTasks.append(task)
# create the poolshare, if any, and affect it to the node
if pool:
# FIXME nodes[0] may not be the root node of the graph...
PoolShare(None, pool, nodes[0], maxRN)
#
# Process dependencies
#
for rule in self.rules:
rule.processDependencies(dependencies)
for node in nodes:
assert isinstance(node.id, int)
self.nodes[node.id] = node
return nodes
def _createTaskGroupFromJSON(self, taskGroupDefinition, user):
# name, parent, arguments, environment, priority, dispatchKey, strategy
id = None
name = taskGroupDefinition['name']
parent = None
arguments = taskGroupDefinition['arguments']
environment = taskGroupDefinition['environment']
requirements = taskGroupDefinition['requirements']
maxRN = taskGroupDefinition['maxRN']
priority = taskGroupDefinition['priority']
dispatchKey = taskGroupDefinition['dispatchKey']
strategy = taskGroupDefinition['strategy']
strategy = loadStrategyClass(strategy.encode())
strategy = strategy()
tags = taskGroupDefinition['tags']
return TaskGroup(id, name, parent, user, arguments, environment, requirements,
maxRN, priority, dispatchKey, strategy, tags=tags)
def _createTaskFromJSON(self, taskDefinition, user):
# id, name, parent, user, priority, dispatchKey, runner, arguments,
# validationExpression, commands, requirements=[], minNbCores=1,
# maxNbCores=0, ramUse=0, environment={}
name = taskDefinition['name']
runner = taskDefinition['runner']
arguments = taskDefinition['arguments']
environment = taskDefinition['environment']
requirements = taskDefinition['requirements']
maxRN = taskDefinition['maxRN']
priority = taskDefinition['priority']
dispatchKey = taskDefinition['dispatchKey']
validationExpression = taskDefinition['validationExpression']
minNbCores = taskDefinition['minNbCores']
maxNbCores = taskDefinition['maxNbCores']
ramUse = taskDefinition['ramUse']
lic = taskDefinition['lic']
tags = taskDefinition['tags']
task = Task(None, name, None, user, maxRN, priority, dispatchKey, runner,
arguments, validationExpression, [], requirements, minNbCores,
maxNbCores, ramUse, environment, lic=lic, tags=tags)
for commandDef in taskDefinition['commands']:
description = commandDef['description']
arguments = commandDef['arguments']
task.commands.append(Command(None, description, task, arguments))
return task
## Resets the lists of elements to create or update in the database.
#
def resetDbElements(self):
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
## Recalculates the max ids of all elements. Generally called after a reload from db.
#
def recomputeMaxIds(self):
self.nodeMaxId = max([n.id for n in self.nodes.values()]) if self.nodes else 0
self.poolMaxId = max([p.id for p in self.pools.values()]) if self.pools else 0
self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()]) if self.renderNodes else 0
self.taskMaxId = max([t.id for t in self.tasks.values()]) if self.tasks else 0
self.commandMaxId = max([c.id for c in self.commands.values()]) if self.commands else 0
self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()]) if self.poolShares else 0
## Removes from the dispatchtree the provided element and all its parents and children.
#
def unregisterElementsFromTree(self, element):
# /////////////// Handling of the Task
if isinstance(element, Task):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for cmd in element.commands:
self.unregisterElementsFromTree(cmd)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskGroup
elif isinstance(element, TaskGroup):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for task in element.tasks:
self.unregisterElementsFromTree(task)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskNode
elif isinstance(element, TaskNode):
# remove the element from the children of the parent
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the FolderNode
elif isinstance(element, FolderNode):
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the Command
elif isinstance(element, Command):
del self.commands[element.id]
self.toArchiveElements.append(element)
### methods called after interaction with a Task
def onTaskCreation(self, task):
if task.id == None:
self.taskMaxId += 1
task.id = self.taskMaxId
self.toCreateElements.append(task)
else:
self.taskMaxId = max(self.taskMaxId, task.id)
self.tasks[task.id] = task
def onTaskDestruction(self, task):
self.unregisterElementsFromTree(task)
def onTaskChange(self, task, field, oldvalue, newvalue):
self.toModifyElements.append(task)
### methods called after interaction with a BaseNode
def onNodeCreation(self, node):
if node.id == None:
self.nodeMaxId += 1
node.id = self.nodeMaxId
self.toCreateElements.append(node)
else:
self.nodeMaxId = max(self.nodeMaxId, node.id)
if node.parent == None:
node.parent = self.root
def onNodeDestruction(self, node):
del self.nodes[node.id]
def onNodeChange(self, node, field, oldvalue, newvalue):
# FIXME: do something when nodes are reparented from or to the root node
if node.id is not None:
self.toModifyElements.append(node)
if field == "status" and node.reverseDependencies:
self.modifiedNodes.append(node)
### methods called after interaction with a RenderNode
def onRenderNodeCreation(self, renderNode):
if renderNode.id == None:
self.renderNodeMaxId += 1
renderNode.id = self.renderNodeMaxId
self.toCreateElements.append(renderNode)
else:
self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id)
self.renderNodes[renderNode.name] = renderNode
def onRenderNodeDestruction(self, rendernode):
del self.renderNodes[rendernode.name]
self.toArchiveElements.append(rendernode)
def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue):
self.toModifyElements.append(rendernode)
### methods called after interaction with a Pool
def onPoolCreation(self, pool):
if pool.id == None:
self.poolMaxId += 1
pool.id = self.poolMaxId
self.toCreateElements.append(pool)
else:
self.poolMaxId = max(self.poolMaxId, pool.id)
self.pools[pool.name] = pool
def onPoolDestruction(self, pool):
del self.pools[pool.name]
self.toArchiveElements.append(pool)
def onPoolChange(self, pool, field, oldvalue, newvalue):
if pool not in self.toModifyElements:
self.toModifyElements.append(pool)
### methods called after interaction with a Command
def onCommandCreation(self, command):
if command.id is None:
self.commandMaxId += 1
command.id = self.commandMaxId
self.toCreateElements.append(command)
else:
self.commandMaxId = max(self.commandMaxId, command.id)
self.commands[command.id] = command
def onCommandChange(self, command, field, oldvalue, newvalue):
self.toModifyElements.append(command)
for node in command.task.nodes.values():
node.invalidate()
### methods called after interaction with a Pool
def onPoolShareCreation(self, poolShare):
if poolShare.id is None:
self.poolShareMaxId += 1
poolShare.id = self.poolShareMaxId
self.toCreateElements.append(poolShare)
else:
self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id)
self.poolShares[poolShare.id] = poolShare
def clear(self):
self.queue.clear()
return WeakValueDictionary.clear(self)
class TaskQueue(AbstractTaskQueue):
"""Simple in-memory task queue implementation"""
@classmethod
def factory(cls, url, name=const.DEFAULT, *args, **kw):
obj = _REFS.get((url, name))
if obj is None:
obj = _REFS[(url, name)] = cls(url, name, *args, **kw)
return obj
def __init__(self, *args, **kw):
super(TaskQueue, self).__init__(*args, **kw)
self.queue = Queue()
self.results = WeakValueDictionary()
self.results_lock = Lock()
def _init_result(self, result, status, message):
with self.results_lock:
if result.id in self.results:
return False
self.results[result.id] = result
result.__status = status
result.__value = Queue()
result.__task = message
result.__args = {}
result.__lock = Lock()
result.__for = None
return True
def enqueue_task(self, result, message):
if self._init_result(result, const.ENQUEUED, message):
self.queue.put(result)
return True
return False
def defer_task(self, result, message, args):
if self._init_result(result, const.PENDING, message):
results = self.results
# keep references to results to prevent GC
result.__refs = [results.get(arg) for arg in args]
return True
return False
def undefer_task(self, task_id):
result = self.results[task_id]
self.queue.put(result)
def get(self, timeout=None):
try:
result = self.queue.get(timeout=timeout)
except Empty:
return None
result.__status = const.PROCESSING
return result.id, result.__task
def size(self):
return len(self.results)
def discard_pending(self):
with self.results_lock:
while True:
try:
self.queue.get_nowait()
except Empty:
break
self.results.clear()
def reserve_argument(self, argument_id, deferred_id):
result = self.results.get(argument_id)
if result is None:
return (False, None)
with result.__lock:
if result.__for is not None:
return (False, None)
result.__for = deferred_id
try:
message = result.__value.get_nowait()
except Empty:
message = None
if message is not None:
with self.results_lock:
self.results.pop(argument_id, None)
return (True, message)
def set_argument(self, task_id, argument_id, message):
result = self.results[task_id]
with self.results_lock:
self.results.pop(argument_id, None)
with result.__lock:
result.__args[argument_id] = message
return len(result.__args) == len(result.__refs)
def get_arguments(self, task_id):
try:
return self.results[task_id].__args
except KeyError:
return {}
def set_task_timeout(self, task_id, timeout):
pass
def get_status(self, task_id):
result = self.results.get(task_id)
return None if result is None else result.__status
def set_result(self, task_id, message, timeout):
result = self.results.get(task_id)
if result is not None:
with result.__lock:
result.__value.put(message)
return result.__for
def pop_result(self, task_id, timeout):
result = self.results.get(task_id)
if result is None:
return const.TASK_EXPIRED
# with result.__lock:
# if result.__for is not None:
# raise NotImplementedError
# #return const.RESERVED
# result.__for = task_id
try:
if timeout == 0:
value = result.__value.get_nowait()
else:
value = result.__value.get(timeout=timeout)
except Empty:
value = None
else:
self.results.pop(task_id, None)
return value
def discard_result(self, task_id, task_expired_token):
result = self.results.pop(task_id)
if result is not None:
result.__value.put(task_expired_token)
class Signal(object):
def __init__(self, *args):
self.__slots = WeakValueDictionary()
for slot in args:
self.connect(slot)
def __call__(self, slot, *args, **kwargs):
"""
Emit signal. If slot passed signal will be called only for this
slot, for all connected slots otherwise.
Calling this method directly lead to immediate signal processing.
It may be not thread-safe. Use emit method from this module for
delayed calling of signals.
"""
if slot is not None:
slots = (self.__slots[self.key(slot)], )
else:
slots = self.__slots.values()
for func in slots:
func(*args, **kwargs)
def key(self, slot):
"""
Get local key name for slot.
"""
if type(slot) == types.FunctionType:
key = (slot.__module__, slot.__name__)
elif type(slot) == types.MethodType:
key = (slot.__func__, id(slot.__self__))
elif isinstance(slot, basestring):
if not slot in registred_slots.keys():
raise ValueError('Slot {0} does not exists.'.format(slot))
key = slot
else:
raise ValueError('Slot {0} has non-slot type'.format(slot))
return key
def connect(self, slot):
"""
Connect signal to slot. Slot may be function, instance method
or name of function perviously registred by `slot` decorator.
"""
key = self.key(slot)
if type(slot) == types.FunctionType:
self.__slots[key] = slot
elif type(slot) == types.MethodType:
self.__slots[key] = partial(slot.__func__, slot.__self__)
elif isinstance(slot, basestring):
self.__slots[key] = registred_slots[slot]
def disconnect(self, slot):
"""
Remove slot from signal connetions.
"""
key = self.key(slot)
del self.__slots[key]
def clear(self):
"""
Disconnect all slots from signal.
"""
self.__slots.clear()
class SqlStore(Store):
"""
Save and load objects in a SQL db, using python's `pickle` module
to serialize objects into a specific field.
Access to the DB is done via SQLAlchemy module, therefore any
driver supported by SQLAlchemy will be supported by this class.
The `url` argument is used to access the store. It is supposed to
be a `gc3libs.url.Url`:class: class, and therefore may contain
username, password, host and port if they are needed by the db
used.
The `table_name` argument is the name of the table to create. By
default it's ``store``. Alternatively, the table name can be
given in the "fragment" part of the database URL, as
``#table=...`` (replace ``...`` with the actual table name). The
constructor argument takes precedence over the table name
specified in the DB URL.
The constructor will create the `table_name` table if it does not
exist, but if there already is such a table it will assume that
its schema is compatible with our needs. A minimal table schema
is as follows::
+-----------+--------------+------+-----+---------+
| Field | Type | Null | Key | Default |
+-----------+--------------+------+-----+---------+
| id | int(11) | NO | PRI | NULL |
| data | blob | YES | | NULL |
| state | varchar(128) | YES | | NULL |
+-----------+--------------+------+-----+---------+
The meaning of the fields is:
- `id`: this is the id returned by the `save()` method and
uniquely identifies a stored object.
- `data`: serialized Python object.
- `state`: if the object is a `Task`:class: instance, this will be
its current execution state.
The `extra_fields` constructor argument is used to extend the
database. It must contain a mapping `*column*: *function*`
where:
- *column* is a `sqlalchemy.Column` object.
- *function* is a function which takes the object to be saved as
argument and returns the value to be stored into the
database. Any exception raised by this function will be
*ignored*. Classes `GetAttribute`:class: and `GetItem`:class:
in module `get`:mod: provide convenient helpers to save object
attributes into table columns.
For each extra column the `save()` method will call the
corresponding *function* in order to get the correct value to
store into the DB.
Any extra keyword arguments are ignored for compatibility with
`FilesystemStore`:class:.
"""
def __init__(self,
url,
table_name=None,
idfactory=None,
extra_fields=None,
create=True,
**extra_args):
"""
Open a connection to the storage database identified by `url`.
DB backend (MySQL, psql, sqlite3) is chosen based on the
`url.scheme` value.
"""
super(SqlStore, self).__init__(url)
if self.url.fragment:
kv = parse_qs(self.url.fragment)
else:
kv = {}
# init static public args
self.idfactory = idfactory or IdFactory(id_class=IntId)
url_table_names = kv.get('table')
if url_table_names:
url_table_name = url_table_names[-1] # last wins
else:
url_table_name = ''
if table_name is None:
self.table_name = url_table_name or "store"
else:
if table_name != url_table_name:
gc3libs.log.debug(
"DB table name given in store URL fragment,"
" but overriden by `table` argument to SqlStore()")
self.table_name = table_name
# save ctor args for lazy-initialization
self._init_extra_fields = (extra_fields
if extra_fields is not None else {})
self._init_create = create
# create slots for lazy-init'ed attrs
self._real_engine = None
self._real_extra_fields = None
self._real_tables = None
self._loaded = WeakValueDictionary()
@staticmethod
def _to_sqlalchemy_url(url):
if url.scheme == 'sqlite':
# rewrite ``sqlite`` URLs to be RFC compliant, see:
# https://github.com/uzh/gc3pie/issues/261
db_url = "%s://%s/%s" % (url.scheme, url.netloc, url.path)
else:
db_url = str(url)
# remove fragment identifier, if any
try:
fragment_loc = db_url.index('#')
db_url = db_url[:fragment_loc]
except ValueError:
pass
return db_url
def _delayed_init(self):
"""
Perform initialization tasks that can interfere with
forking/multiprocess setup.
See `GC3Pie issue #550 <https://github.com/uzh/gc3pie/issues/550>`_
for more details and motivation.
"""
url = self._to_sqlalchemy_url(self.url)
gc3libs.log.debug("Initializing SQLAlchemy engine for `%s`...", url)
self._real_engine = sqla.create_engine(url)
# create schema
meta = sqla.MetaData(bind=self._real_engine)
table = sqla.Table(
self.table_name, meta,
sqla.Column('id', sqla.Integer(),
primary_key=True, nullable=False),
sqla.Column('data', sqla.LargeBinary()),
sqla.Column('state', sqla.String(length=128)))
# create internal rep of table
self._real_extra_fields = {}
for col, func in self._init_extra_fields.items():
assert isinstance(col, sqla.Column)
table.append_column(col.copy())
self._real_extra_fields[col.name] = func
# check if the db exists and already has a 'store' table
current_meta = sqla.MetaData(bind=self._real_engine)
current_meta.reflect()
if self._init_create and self.table_name not in current_meta.tables:
meta.create_all()
self._real_tables = meta.tables[self.table_name]
def pre_fork(self):
"""
Dispose current SQLAlchemy engine (if any).
A new SQLAlchemy engine will be initialized
upon the next interaction with a DB.
This method only exists to allow `SessionBasedDaemon`:class:
and similar applications that can do DB operations after
fork()ing to continue to operate, without incurring into a
SQLAlchemy "OperationalError: (...) could not receive data
from server: Transport endpoint is not connected"
"""
if self._real_engine:
self._real_engine.dispose()
self._real_engine = None
self._real_extra_fields = None
self._real_tables = None
@property
def _engine(self):
if self._real_engine is None:
self._delayed_init()
return self._real_engine
@property
def _tables(self):
if self._real_tables is None:
self._delayed_init()
return self._real_tables
# FIXME: Remove once the TissueMAPS code is updated not to use this any more!
@property
def t_store(self):
"""
Deprecated compatibility alias for `SqlStore._tables`
"""
warn(
"`SqlStore.t_store` has been renamed to `SqlStore._tables`;"
" please update your code", DeprecationWarning, 2)
return self._tables
@property
def extra_fields(self):
if self._real_extra_fields is None:
self._delayed_init()
return self._real_extra_fields
@same_docstring_as(Store.invalidate_cache)
def invalidate_cache(self):
self._loaded.clear()
@same_docstring_as(Store.list)
def list(self):
q = sql.select([self._tables.c.id])
with self._engine.begin() as conn:
rows = conn.execute(q)
ids = [i[0] for i in rows.fetchall()]
return ids
@same_docstring_as(Store.replace)
def replace(self, id_, obj):
self._save_or_replace(id_, obj)
# copied from FilesystemStore
@same_docstring_as(Store.save)
def save(self, obj):
if not hasattr(obj, 'persistent_id'):
obj.persistent_id = self.idfactory.new(obj)
return self._save_or_replace(obj.persistent_id, obj)
def _save_or_replace(self, id_, obj):
# if __debug__:
# global _lvl
# _lvl += '>'
# gc3libs.log.debug("%s Saving %r@%x as %s ...", _lvl, obj, id(obj), id_)
# build row to insert/update
fields = {'id': id_}
with closing(BytesIO()) as dstdata:
make_pickler(self, dstdata, obj).dump(obj)
fields['data'] = dstdata.getvalue()
try:
fields['state'] = obj.execution.state
except AttributeError:
# If we cannot determine the state of a task, consider it UNKNOWN.
fields['state'] = Run.State.UNKNOWN
# insert into db
for column in self.extra_fields:
try:
fields[column] = self.extra_fields[column](obj)
except Exception as ex:
gc3libs.log.warning(
"Error saving DB column '%s' of object '%s': %s: %s",
column, obj, ex.__class__.__name__, str(ex))
if __debug__:
for column in fields:
if column == 'data':
continue
gc3libs.log.debug(
"Writing value '%s' in column '%s' for object '%s'",
fields[column], column, obj)
q = sql.select([self._tables.c.id]).where(self._tables.c.id == id_)
with self._engine.begin() as conn:
r = conn.execute(q)
if not r.fetchone():
# It's an insert
q = self._tables.insert().values(**fields)
else:
# it's an update
q = self._tables.update().where(
self._tables.c.id == id_).values(**fields)
conn.execute(q)
obj.persistent_id = id_
if hasattr(obj, 'changed'):
obj.changed = False
# update cache
if str(id_) in self._loaded:
old = self._loaded[str(id_)]
if old is not obj:
self._loaded[str(id_)] = obj
# if __debug__:
# gc3libs.log.debug(
# "%s Overwriting object %s %r@%x with %r@%x",
# _lvl, id_, old, id(old), obj, id(obj))
# from traceback import format_stack
# gc3libs.log.debug("Traceback:\n%s", ''.join(format_stack()))
# if __debug__:
# gc3libs.log.debug("%s Done saving %r@%x as %s ...", _lvl, obj, id(obj), id_)
# if _lvl:
# _lvl = _lvl[:-1]
# return id
return id_
@same_docstring_as(Store.load)
def load(self, id_):
# if __debug__:
# global _lvl
# _lvl += '<'
# gc3libs.log.debug("%s Store %s: Loading task %s %r ...", _lvl, self, id_, type(id_))
# return cached copy, if any
try:
obj = self._loaded[str(id_)]
# if __debug__:
# if _lvl:
# _lvl = _lvl[:-1]
# gc3libs.log.debug("%s Store %s: Returning cached object %r@%x as task %s", _lvl, self, obj, id(obj), id_)
return obj
except KeyError:
pass
# no cached copy, load from disk
q = sql.select([self._tables.c.data]).where(self._tables.c.id == id_)
with self._engine.begin() as conn:
rawdata = conn.execute(q).fetchone()
if not rawdata:
raise gc3libs.exceptions.LoadError(
"Unable to find any object with ID '%s'" % id_)
obj = make_unpickler(self, BytesIO(rawdata[0])).load()
super(SqlStore, self)._update_to_latest_schema()
assert str(id_) not in self._loaded
self._loaded[str(id_)] = obj
# if __debug__:
# if _lvl:
# _lvl = _lvl[:-1]
# gc3libs.log.debug("%s Store %s: Done loading task %s as %r@%x.", _lvl, self, id_, obj, id(obj))
return obj
@same_docstring_as(Store.remove)
def remove(self, id_):
with self._engine.begin() as conn:
conn.execute(self._tables.delete().where(self._tables.c.id == id_))
try:
del self._loaded[str(id_)]
except KeyError:
pass
class DispatchTree(object):
def __init__(self):
# core data
self.root = FolderNode(0, "root", None, "root", 1, 1, 0,
FifoStrategy())
self.nodes = WeakValueDictionary()
self.nodes[0] = self.root
self.pools = {}
self.renderNodes = {}
self.tasks = {}
self.rules = []
self.poolShares = {}
self.commands = {}
# deduced properties
self.nodeMaxId = 0
self.poolMaxId = 0
self.renderNodeMaxId = 0
self.taskMaxId = 0
self.commandMaxId = 0
self.poolShareMaxId = 0
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
# listeners
self.nodeListener = ObjectListener(self.onNodeCreation,
self.onNodeDestruction,
self.onNodeChange)
self.taskListener = ObjectListener(self.onTaskCreation,
self.onTaskDestruction,
self.onTaskChange)
# # JSA
# self.taskGroupListener = ObjectListener(self.onTaskCreation, self.onTaskDestruction, self.onTaskGroupChange)
self.renderNodeListener = ObjectListener(self.onRenderNodeCreation,
self.onRenderNodeDestruction,
self.onRenderNodeChange)
self.poolListener = ObjectListener(self.onPoolCreation,
self.onPoolDestruction,
self.onPoolChange)
self.commandListener = ObjectListener(
onCreationEvent=self.onCommandCreation,
onChangeEvent=self.onCommandChange)
self.poolShareListener = ObjectListener(self.onPoolShareCreation)
self.modifiedNodes = []
def registerModelListeners(self):
BaseNode.changeListeners.append(self.nodeListener)
Task.changeListeners.append(self.taskListener)
TaskGroup.changeListeners.append(self.taskListener)
RenderNode.changeListeners.append(self.renderNodeListener)
Pool.changeListeners.append(self.poolListener)
Command.changeListeners.append(self.commandListener)
PoolShare.changeListeners.append(self.poolShareListener)
def destroy(self):
BaseNode.changeListeners.remove(self.nodeListener)
Task.changeListeners.remove(self.taskListener)
RenderNode.changeListeners.remove(self.renderNodeListener)
Pool.changeListeners.remove(self.poolListener)
Command.changeListeners.remove(self.commandListener)
PoolShare.changeListeners.remove(self.poolShareListener)
self.root = None
self.nodes.clear()
self.pools.clear()
self.renderNodes.clear()
self.tasks.clear()
self.rules = None
self.commands.clear()
self.poolShares = None
self.modifiedNodes = None
self.toCreateElements = None
self.toModifyElements = None
self.toArchiveElements = None
def findNodeByPath(self, path, default=None):
nodenames = splitpath(path)
node = self.root
for name in nodenames:
for child in node.children:
if child.name == name:
node = child
break
else:
return default
return node
def updateCompletionAndStatus(self):
self.root.updateCompletionAndStatus()
def validateDependencies(self):
nodes = set()
for dependency in self.modifiedNodes:
for node in dependency.reverseDependencies:
nodes.add(node)
del self.modifiedNodes[:]
for node in nodes:
# logger.debug("Dependencies on %r = %r"% (node.name, node.checkDependenciesSatisfaction() ) )
if not hasattr(node, "task") or node.task is None:
continue
if isinstance(node, TaskNode):
if node.checkDependenciesSatisfaction():
for cmd in node.task.commands:
if cmd.status == CMD_BLOCKED:
cmd.status = CMD_READY
else:
for cmd in node.task.commands:
if cmd.status == CMD_READY:
cmd.status = CMD_BLOCKED
# TODO: may be needed to check dependencies on task groups
# so far, a hack is done on the client side when submitting:
# dependencies of a taksgroup are reported on each task of its heirarchy
#
# elif isinstance(node, FolderNode):
#
# if node.checkDependenciesSatisfaction():
# for cmd in node.getAllCommands():
# if cmd.status == CMD_BLOCKED:
# cmd.status = CMD_READY
# else:
# for cmd in node.getAllCommands():
# if cmd.status == CMD_READY:
# cmd.status = CMD_BLOCKED
def registerNewGraph(self, graph):
user = graph['user']
taskDefs = graph['tasks']
poolName = graph['poolName']
if 'maxRN' in graph.items():
maxRN = int(graph['maxRN'])
else:
maxRN = -1
#
# Create objects.
#
tasks = [None for i in xrange(len(taskDefs))]
for (index, taskDef) in enumerate(taskDefs):
if taskDef['type'] == 'Task':
# logger.debug("taskDef.watcherPackages = %s" % taskDef["watcherPackages"])
# logger.debug("taskDef.runnerPackages = %s" % taskDef["runnerPackages"])
task = self._createTaskFromJSON(taskDef, user)
elif taskDef['type'] == 'TaskGroup':
task = self._createTaskGroupFromJSON(taskDef, user)
tasks[index] = task
root = tasks[graph['root']]
# get the pool
try:
pool = self.pools[poolName]
except KeyError:
pool = Pool(None, poolName)
self.pools[poolName] = pool
#
# Rebuild full job hierarchy
#
for (taskDef, task) in zip(taskDefs, tasks):
if taskDef['type'] == 'TaskGroup':
for taskIndex in taskDef['tasks']:
task.addTask(tasks[taskIndex])
tasks[taskIndex].parent = task
#
# Compute dependencies for each created task or taskgroup object.
#
dependencies = {}
for (taskDef, task) in zip(taskDefs, tasks):
taskDependencies = {}
if not isinstance(taskDef['dependencies'], list):
raise SyntaxError(
"Dependencies must be a list of (taskId, [status-list]), got %r."
% taskDef['dependencies'])
if not all(((isinstance(i, int) and isinstance(sl, list) and all(
(isinstance(s, int) for s in sl)))
for (i, sl) in taskDef['dependencies'])):
raise SyntaxError(
"Dependencies must be a list of (taskId, [status-list]), got %r."
% taskDef['dependencies'])
for (taskIndex, statusList) in taskDef['dependencies']:
taskDependencies[tasks[taskIndex]] = statusList
dependencies[task] = taskDependencies
#
# Apply rules to generate dispatch tree nodes.
#
if not self.rules:
logger.warning("graph submitted but no rule has been defined")
unprocessedTasks = [root]
nodes = []
while unprocessedTasks:
unprocessedTask = unprocessedTasks.pop(0)
for rule in self.rules:
try:
nodes += rule.apply(unprocessedTask)
except RuleError:
logger.warning("rule %s failed for graph %s" %
(rule, graph))
raise
if isinstance(unprocessedTask, TaskGroup):
for task in unprocessedTask:
unprocessedTasks.append(task)
# create the poolshare, if any, and affect it to the node
if pool:
# FIXME nodes[0] may not be the root node of the graph...
ps = PoolShare(None, pool, nodes[0], maxRN)
# if maxRN is not -1 (e.g not default) set the userDefinedMaxRN to true
if maxRN != -1:
ps.userDefinedMaxRN = True
#
# Process dependencies
#
for rule in self.rules:
rule.processDependencies(dependencies)
for node in nodes:
assert isinstance(node.id, int)
self.nodes[node.id] = node
# Init number of command in hierarchy
self.populateCommandCounts(nodes[0])
return nodes
def populateCommandCounts(self, node):
"""
Updates "commandCount" over a whole hierarchy starting from the given node.
"""
res = 0
if isinstance(node, FolderNode):
for child in node.children:
res += self.populateCommandCounts(child)
elif isinstance(node, TaskNode):
res = len(node.task.commands)
node.commandCount = res
return res
def _createTaskGroupFromJSON(self, taskGroupDefinition, user):
# name, parent, arguments, environment, priority, dispatchKey, strategy
id = None
name = taskGroupDefinition['name']
parent = None
arguments = taskGroupDefinition['arguments']
environment = taskGroupDefinition['environment']
requirements = taskGroupDefinition['requirements']
maxRN = taskGroupDefinition['maxRN']
priority = taskGroupDefinition['priority']
dispatchKey = taskGroupDefinition['dispatchKey']
strategy = taskGroupDefinition['strategy']
strategy = loadStrategyClass(strategy.encode())
strategy = strategy()
tags = taskGroupDefinition['tags']
timer = None
if 'timer' in taskGroupDefinition.keys():
timer = taskGroupDefinition['timer']
return TaskGroup(id,
name,
parent,
user,
arguments,
environment,
requirements,
maxRN,
priority,
dispatchKey,
strategy,
tags=tags,
timer=timer)
def _createTaskFromJSON(self, taskDefinition, user):
# id, name, parent, user, priority, dispatchKey, runner, arguments,
# validationExpression, commands, requirements=[], minNbCores=1,
# maxNbCores=0, ramUse=0, environment={}
name = taskDefinition['name']
runner = taskDefinition['runner']
arguments = taskDefinition['arguments']
environment = taskDefinition['environment']
requirements = taskDefinition['requirements']
maxRN = taskDefinition['maxRN']
priority = taskDefinition['priority']
dispatchKey = taskDefinition['dispatchKey']
validationExpression = taskDefinition['validationExpression']
minNbCores = taskDefinition['minNbCores']
maxNbCores = taskDefinition['maxNbCores']
ramUse = taskDefinition['ramUse']
lic = taskDefinition['lic']
tags = taskDefinition['tags']
runnerPackages = taskDefinition.get('runnerPackages', '')
watcherPackages = taskDefinition.get('watcherPackages', '')
timer = None
if 'timer' in taskDefinition.keys():
timer = taskDefinition['timer']
maxAttempt = taskDefinition.get('maxAttempt', 1)
task = Task(None,
name,
None,
user,
maxRN,
priority,
dispatchKey,
runner,
arguments,
validationExpression, [],
requirements,
minNbCores,
maxNbCores,
ramUse,
environment,
lic=lic,
tags=tags,
timer=timer,
maxAttempt=maxAttempt,
runnerPackages=runnerPackages,
watcherPackages=watcherPackages)
for commandDef in taskDefinition['commands']:
description = commandDef['description']
arguments = commandDef['arguments']
cmd = Command(None,
description,
task,
arguments,
runnerPackages=runnerPackages,
watcherPackages=watcherPackages)
task.commands.append(cmd)
# import sys
# logger.warning("cmd creation : %s" % str(sys.getrefcount(cmd)))
return task
## Resets the lists of elements to create or update in the database.
#
def resetDbElements(self):
self.toCreateElements = []
self.toModifyElements = []
self.toArchiveElements = []
## Recalculates the max ids of all elements. Generally called after a reload from db.
#
def recomputeMaxIds(self):
self.nodeMaxId = max([n.id for n in self.nodes.values()
]) if self.nodes else 0
self.nodeMaxId = max(self.nodeMaxId, StatDB.getFolderNodesMaxId(),
StatDB.getTaskNodesMaxId())
self.poolMaxId = max([p.id for p in self.pools.values()
]) if self.pools else 0
self.poolMaxId = max(self.poolMaxId, StatDB.getPoolsMaxId())
self.renderNodeMaxId = max([rn.id for rn in self.renderNodes.values()
]) if self.renderNodes else 0
self.renderNodeMaxId = max(self.renderNodeMaxId,
StatDB.getRenderNodesMaxId())
self.taskMaxId = max([t.id for t in self.tasks.values()
]) if self.tasks else 0
self.taskMaxId = max(self.taskMaxId, StatDB.getTasksMaxId(),
StatDB.getTaskGroupsMaxId())
self.commandMaxId = max([c.id for c in self.commands.values()
]) if self.commands else 0
self.commandMaxId = max(self.commandMaxId, StatDB.getCommandsMaxId())
self.poolShareMaxId = max([ps.id for ps in self.poolShares.values()
]) if self.poolShares else 0
self.poolShareMaxId = max(self.poolShareMaxId,
StatDB.getPoolSharesMaxId())
## Removes from the dispatchtree the provided element and all its parents and children.
#
def unregisterElementsFromTree(self, element):
# /////////////// Handling of the Task
if isinstance(element, Task):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for cmd in element.commands:
self.unregisterElementsFromTree(cmd)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskGroup
elif isinstance(element, TaskGroup):
del self.tasks[element.id]
self.toArchiveElements.append(element)
for task in element.tasks:
self.unregisterElementsFromTree(task)
for node in element.nodes.values():
self.unregisterElementsFromTree(node)
# /////////////// Handling of the TaskNode
elif isinstance(element, TaskNode):
# remove the element from the children of the parent
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the FolderNode
elif isinstance(element, FolderNode):
if element.parent:
element.parent.removeChild(element)
if element.poolShares:
for poolShare in element.poolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
if element.additionnalPoolShares:
for poolShare in element.additionnalPoolShares.values():
del poolShare.pool.poolShares[poolShare.node]
del self.poolShares[poolShare.id]
self.toArchiveElements.append(poolShare)
del self.nodes[element.id]
self.toArchiveElements.append(element)
for dependency in element.dependencies:
self.unregisterElementsFromTree(dependency)
# /////////////// Handling of the Command
elif isinstance(element, Command):
del self.commands[element.id]
self.toArchiveElements.append(element)
### methods called after interaction with a Task
def onTaskCreation(self, task):
# logger.info(" -- on task creation: %s" % task)
if task.id is None:
self.taskMaxId += 1
task.id = self.taskMaxId
self.toCreateElements.append(task)
else:
self.taskMaxId = max(self.taskMaxId, task.id,
StatDB.getTasksMaxId(),
StatDB.getTaskGroupsMaxId())
self.tasks[task.id] = task
def onTaskDestruction(self, task):
# logger.info(" -- on task destruction: %s" % task)
self.unregisterElementsFromTree(task)
def onTaskChange(self, task, field, oldvalue, newvalue):
"""
Normally, taskgroup should not be updated to DB, there would be too manby updates due to command/state changes
However in order to keep track of comments (stored in task's tags[comment] field), we make the following change:
- enable task/taskgroups update in DB (cf pulidb.py)
- disable changeEvent (append an event in dispatchTree.toModifyElements array) for all fields of tasks and TGs
BUT the only field we want to update: "tags"
"""
if field == "tags":
self.toModifyElements.append(task)
### methods called after interaction with a BaseNode
def onNodeCreation(self, node):
# logger.info(" -- on node creation: %s" % node)
if node.id is None:
self.nodeMaxId += 1
node.id = self.nodeMaxId
self.toCreateElements.append(node)
else:
self.nodeMaxId = max(self.nodeMaxId, node.id,
StatDB.getFolderNodesMaxId(),
StatDB.getTaskNodesMaxId())
if node.parent is None:
node.parent = self.root
def onNodeDestruction(self, node):
# logger.info(" -- on node destruction: %s" % node)
del self.nodes[node.id]
def onNodeChange(self, node, field, oldvalue, newvalue):
# logger.info(" -- on node change: %s [ %s = %s -> %s ]" % (node,field, oldvalue, newvalue) )
# FIXME: do something when nodes are reparented from or to the root node
if node.id is not None:
self.toModifyElements.append(node)
if field == "status" and node.reverseDependencies:
self.modifiedNodes.append(node)
### methods called after interaction with a RenderNode
def onRenderNodeCreation(self, renderNode):
if renderNode.id is None:
self.renderNodeMaxId += 1
renderNode.id = self.renderNodeMaxId
self.toCreateElements.append(renderNode)
else:
self.renderNodeMaxId = max(self.renderNodeMaxId, renderNode.id,
StatDB.getRenderNodesMaxId())
self.renderNodes[renderNode.name] = renderNode
def onRenderNodeDestruction(self, rendernode):
try:
del self.renderNodes[rendernode.name]
self.toArchiveElements.append(rendernode)
except KeyError:
# TOFIX: use of class method vs obj method in changeListener might generate a duplicate call
logger.warning("RN %s seems to have been deleted already." %
rendernode.name)
def onRenderNodeChange(self, rendernode, field, oldvalue, newvalue):
if field == "performance":
self.toModifyElements.append(rendernode)
### methods called after interaction with a Pool
def onPoolCreation(self, pool):
if pool.id is None:
self.poolMaxId += 1
pool.id = self.poolMaxId
self.toCreateElements.append(pool)
else:
self.poolMaxId = max(self.poolMaxId, pool.id,
StatDB.getPoolsMaxId())
self.pools[pool.name] = pool
def onPoolDestruction(self, pool):
del self.pools[pool.name]
self.toArchiveElements.append(pool)
def onPoolChange(self, pool, field, oldvalue, newvalue):
if pool not in self.toModifyElements:
self.toModifyElements.append(pool)
### methods called after interaction with a Command
def onCommandCreation(self, command):
if command.id is None:
self.commandMaxId += 1
command.id = self.commandMaxId
self.toCreateElements.append(command)
else:
self.commandMaxId = max(self.commandMaxId, command.id,
StatDB.getCommandsMaxId())
self.commands[command.id] = command
def onCommandChange(self, command, field, oldvalue, newvalue):
self.toModifyElements.append(command)
if command.task is not None:
for node in command.task.nodes.values():
node.invalidate()
### methods called after interaction with a Pool
def onPoolShareCreation(self, poolShare):
if poolShare.id is None:
self.poolShareMaxId += 1
poolShare.id = self.poolShareMaxId
self.toCreateElements.append(poolShare)
else:
self.poolShareMaxId = max(self.poolShareMaxId, poolShare.id,
StatDB.getPoolSharesMaxId())
self.poolShares[poolShare.id] = poolShare
def clear(self):
self._head = self._tail = None
self._keepDict.clear()
WeakValueDictionary.clear(self)
