def spawn(*prototypes, **kwargs): """ Spawn a number of prototyped objects. Args: prototypes (dict): Each argument should be a prototype dictionary. Kwargs: prototype_modules (str or list): A python-path to a prototype module, or a list of such paths. These will be used to build the global protparents dictionary accessible by the input prototypes. If not given, it will instead look for modules defined by settings.PROTOTYPE_MODULES. prototype_parents (dict): A dictionary holding a custom prototype-parent dictionary. Will overload same-named prototypes from prototype_modules. return_parents (bool): Only return a dict of the prototype-parents (no object creation happens) only_validate (bool): Only run validation of prototype/parents (no object creation) and return the create-kwargs. Returns: object (Object, dict or list): Spawned object(s). If `only_validate` is given, return a list of the creation kwargs to build the object(s) without actually creating it. If `return_parents` is set, instead return dict of prototype parents. """ # get available protparents protparents = { prot['prototype_key'].lower(): prot for prot in protlib.search_prototype() } if not kwargs.get("only_validate"): # homogenization to be more lenient about prototype format when entering the prototype manually prototypes = [ protlib.homogenize_prototype(prot) for prot in prototypes ] # overload module's protparents with specifically given protparents # we allow prototype_key to be the key of the protparent dict, to allow for module-level # prototype imports. We need to insert prototype_key in this case for key, protparent in kwargs.get("prototype_parents", {}).items(): key = str(key).lower() protparent['prototype_key'] = str(protparent.get("prototype_key", key)).lower() protparents[key] = protparent if "return_parents" in kwargs: # only return the parents return copy.deepcopy(protparents) objsparams = [] for prototype in prototypes: protlib.validate_prototype(prototype, None, protparents, is_prototype_base=True) prot = _get_prototype( prototype, protparents, uninherited={"prototype_key": prototype.get("prototype_key")}) if not prot: continue # extract the keyword args we need to create the object itself. If we get a callable, # call that to get the value (don't catch errors) create_kwargs = {} # we must always add a key, so if not given we use a shortened md5 hash. There is a (small) # chance this is not unique but it should usually not be a problem. val = prot.pop( "key", "Spawned-{}".format(hashlib.md5(str(time.time())).hexdigest()[:6])) create_kwargs["db_key"] = init_spawn_value(val, str) val = prot.pop("location", None) create_kwargs["db_location"] = init_spawn_value(val, value_to_obj) val = prot.pop("home", settings.DEFAULT_HOME) create_kwargs["db_home"] = init_spawn_value(val, value_to_obj) val = prot.pop("destination", None) create_kwargs["db_destination"] = init_spawn_value(val, value_to_obj) val = prot.pop("typeclass", settings.BASE_OBJECT_TYPECLASS) create_kwargs["db_typeclass_path"] = init_spawn_value(val, str) # extract calls to handlers val = prot.pop("permissions", []) permission_string = init_spawn_value(val, make_iter) val = prot.pop("locks", "") lock_string = init_spawn_value(val, str) val = prot.pop("aliases", []) alias_string = init_spawn_value(val, make_iter) val = prot.pop("tags", []) tags = [] for (tag, category, data) in val: tags.append((init_spawn_value(tag, str), category, data)) prototype_key = prototype.get('prototype_key', None) if prototype_key: # we make sure to add a tag identifying which prototype created this object tags.append((prototype_key, _PROTOTYPE_TAG_CATEGORY)) val = prot.pop("exec", "") execs = init_spawn_value(val, make_iter) # extract ndb assignments nattributes = dict( (key.split("_", 1)[1], init_spawn_value(val, value_to_obj)) for key, val in prot.items() if key.startswith("ndb_")) # the rest are attribute tuples (attrname, value, category, locks) val = make_iter(prot.pop("attrs", [])) attributes = [] for (attrname, value, category, locks) in val: attributes.append( (attrname, init_spawn_value(value), category, locks)) simple_attributes = [] for key, value in ((key, value) for key, value in prot.items() if not (key.startswith("ndb_"))): # we don't support categories, nor locks for simple attributes if key in _PROTOTYPE_META_NAMES: continue else: simple_attributes.append( (key, init_spawn_value(value, value_to_obj_or_any), None, None)) attributes = attributes + simple_attributes attributes = [ tup for tup in attributes if not tup[0] in _NON_CREATE_KWARGS ] # pack for call into _batch_create_object objsparams.append((create_kwargs, permission_string, lock_string, alias_string, nattributes, attributes, tags, execs)) if kwargs.get("only_validate"): return objsparams return batch_create_object(*objsparams)
def batch_update_objects_with_prototype(prototype, diff=None, objects=None): """ Update existing objects with the latest version of the prototype. Args: prototype (str or dict): Either the `prototype_key` to use or the prototype dict itself. diff (dict, optional): This a diff structure that describes how to update the protototype. If not given this will be constructed from the first object found. objects (list, optional): List of objects to update. If not given, query for these objects using the prototype's `prototype_key`. Returns: changed (int): The number of objects that had changes applied to them. """ prototype = protlib.homogenize_prototype(prototype) if isinstance(prototype, basestring): new_prototype = protlib.search_prototype(prototype) else: new_prototype = prototype prototype_key = new_prototype['prototype_key'] if not objects: objects = ObjectDB.objects.get_by_tag(prototype_key, category=_PROTOTYPE_TAG_CATEGORY) if not objects: return 0 if not diff: diff, _ = prototype_diff_from_object(new_prototype, objects[0]) # make sure the diff is flattened diff = flatten_diff(diff) changed = 0 for obj in objects: do_save = False old_prot_key = obj.tags.get(category=_PROTOTYPE_TAG_CATEGORY, return_list=True) old_prot_key = old_prot_key[0] if old_prot_key else None if prototype_key != old_prot_key: obj.tags.clear(category=_PROTOTYPE_TAG_CATEGORY) obj.tags.add(prototype_key, category=_PROTOTYPE_TAG_CATEGORY) for key, directive in diff.items(): if directive in ('UPDATE', 'REPLACE'): if key in _PROTOTYPE_META_NAMES: # prototype meta keys are not stored on-object continue val = new_prototype[key] do_save = True if key == 'key': obj.db_key = init_spawn_value(val, str) elif key == 'typeclass': obj.db_typeclass_path = init_spawn_value(val, str) elif key == 'location': obj.db_location = init_spawn_value(val, value_to_obj) elif key == 'home': obj.db_home = init_spawn_value(val, value_to_obj) elif key == 'destination': obj.db_destination = init_spawn_value(val, value_to_obj) elif key == 'locks': if directive == 'REPLACE': obj.locks.clear() obj.locks.add(init_spawn_value(val, str)) elif key == 'permissions': if directive == 'REPLACE': obj.permissions.clear() obj.permissions.batch_add(*(init_spawn_value(perm, str) for perm in val)) elif key == 'aliases': if directive == 'REPLACE': obj.aliases.clear() obj.aliases.batch_add(*(init_spawn_value(alias, str) for alias in val)) elif key == 'tags': if directive == 'REPLACE': obj.tags.clear() obj.tags.batch_add(*((init_spawn_value(ttag, str), tcategory, tdata) for ttag, tcategory, tdata in val)) elif key == 'attrs': if directive == 'REPLACE': obj.attributes.clear() obj.attributes.batch_add( *((init_spawn_value(akey, str), init_spawn_value(aval, value_to_obj), acategory, alocks) for akey, aval, acategory, alocks in val)) elif key == 'exec': # we don't auto-rerun exec statements, it would be huge security risk! pass else: obj.attributes.add(key, init_spawn_value(val, value_to_obj)) elif directive == 'REMOVE': do_save = True if key == 'key': obj.db_key = '' elif key == 'typeclass': # fall back to default obj.db_typeclass_path = settings.BASE_OBJECT_TYPECLASS elif key == 'location': obj.db_location = None elif key == 'home': obj.db_home = None elif key == 'destination': obj.db_destination = None elif key == 'locks': obj.locks.clear() elif key == 'permissions': obj.permissions.clear() elif key == 'aliases': obj.aliases.clear() elif key == 'tags': obj.tags.clear() elif key == 'attrs': obj.attributes.clear() elif key == 'exec': # we don't auto-rerun exec statements, it would be huge security risk! pass else: obj.attributes.remove(key) if do_save: changed += 1 obj.save() return changed
def batch_update_objects_with_prototype(prototype, diff=None, objects=None): """ Update existing objects with the latest version of the prototype. Args: prototype (str or dict): Either the `prototype_key` to use or the prototype dict itself. diff (dict, optional): This a diff structure that describes how to update the protototype. If not given this will be constructed from the first object found. objects (list, optional): List of objects to update. If not given, query for these objects using the prototype's `prototype_key`. Returns: changed (int): The number of objects that had changes applied to them. """ prototype = protlib.homogenize_prototype(prototype) if isinstance(prototype, basestring): new_prototype = protlib.search_prototype(prototype) else: new_prototype = prototype prototype_key = new_prototype['prototype_key'] if not objects: objects = ObjectDB.objects.get_by_tag(prototype_key, category=_PROTOTYPE_TAG_CATEGORY) if not objects: return 0 if not diff: diff, _ = prototype_diff_from_object(new_prototype, objects[0]) # make sure the diff is flattened diff = flatten_diff(diff) changed = 0 for obj in objects: do_save = False old_prot_key = obj.tags.get(category=_PROTOTYPE_TAG_CATEGORY, return_list=True) old_prot_key = old_prot_key[0] if old_prot_key else None if prototype_key != old_prot_key: obj.tags.clear(category=_PROTOTYPE_TAG_CATEGORY) obj.tags.add(prototype_key, category=_PROTOTYPE_TAG_CATEGORY) for key, directive in diff.items(): if directive in ('UPDATE', 'REPLACE'): if key in _PROTOTYPE_META_NAMES: # prototype meta keys are not stored on-object continue val = new_prototype[key] do_save = True if key == 'key': obj.db_key = init_spawn_value(val, str) elif key == 'typeclass': obj.db_typeclass_path = init_spawn_value(val, str) elif key == 'location': obj.db_location = init_spawn_value(val, value_to_obj) elif key == 'home': obj.db_home = init_spawn_value(val, value_to_obj) elif key == 'destination': obj.db_destination = init_spawn_value(val, value_to_obj) elif key == 'locks': if directive == 'REPLACE': obj.locks.clear() obj.locks.add(init_spawn_value(val, str)) elif key == 'permissions': if directive == 'REPLACE': obj.permissions.clear() obj.permissions.batch_add(*(init_spawn_value(perm, str) for perm in val)) elif key == 'aliases': if directive == 'REPLACE': obj.aliases.clear() obj.aliases.batch_add(*(init_spawn_value(alias, str) for alias in val)) elif key == 'tags': if directive == 'REPLACE': obj.tags.clear() obj.tags.batch_add(*( (init_spawn_value(ttag, str), tcategory, tdata) for ttag, tcategory, tdata in val)) elif key == 'attrs': if directive == 'REPLACE': obj.attributes.clear() obj.attributes.batch_add(*( (init_spawn_value(akey, str), init_spawn_value(aval, value_to_obj), acategory, alocks) for akey, aval, acategory, alocks in val)) elif key == 'exec': # we don't auto-rerun exec statements, it would be huge security risk! pass else: obj.attributes.add(key, init_spawn_value(val, value_to_obj)) elif directive == 'REMOVE': do_save = True if key == 'key': obj.db_key = '' elif key == 'typeclass': # fall back to default obj.db_typeclass_path = settings.BASE_OBJECT_TYPECLASS elif key == 'location': obj.db_location = None elif key == 'home': obj.db_home = None elif key == 'destination': obj.db_destination = None elif key == 'locks': obj.locks.clear() elif key == 'permissions': obj.permissions.clear() elif key == 'aliases': obj.aliases.clear() elif key == 'tags': obj.tags.clear() elif key == 'attrs': obj.attributes.clear() elif key == 'exec': # we don't auto-rerun exec statements, it would be huge security risk! pass else: obj.attributes.remove(key) if do_save: changed += 1 obj.save() return changed
def spawn(*prototypes, **kwargs): """ Spawn a number of prototyped objects. Args: prototypes (dict): Each argument should be a prototype dictionary. Kwargs: prototype_modules (str or list): A python-path to a prototype module, or a list of such paths. These will be used to build the global protparents dictionary accessible by the input prototypes. If not given, it will instead look for modules defined by settings.PROTOTYPE_MODULES. prototype_parents (dict): A dictionary holding a custom prototype-parent dictionary. Will overload same-named prototypes from prototype_modules. return_parents (bool): Only return a dict of the prototype-parents (no object creation happens) only_validate (bool): Only run validation of prototype/parents (no object creation) and return the create-kwargs. Returns: object (Object, dict or list): Spawned object(s). If `only_validate` is given, return a list of the creation kwargs to build the object(s) without actually creating it. If `return_parents` is set, instead return dict of prototype parents. """ # get available protparents protparents = {prot['prototype_key'].lower(): prot for prot in protlib.search_prototype()} # overload module's protparents with specifically given protparents # we allow prototype_key to be the key of the protparent dict, to allow for module-level # prototype imports. We need to insert prototype_key in this case for key, protparent in kwargs.get("prototype_parents", {}).items(): key = str(key).lower() protparent['prototype_key'] = str(protparent.get("prototype_key", key)).lower() protparents[key] = protparent if "return_parents" in kwargs: # only return the parents return copy.deepcopy(protparents) objsparams = [] for prototype in prototypes: protlib.validate_prototype(prototype, None, protparents, is_prototype_base=True) prot = _get_prototype(prototype, protparents, uninherited={"prototype_key": prototype.get("prototype_key")}) if not prot: continue # extract the keyword args we need to create the object itself. If we get a callable, # call that to get the value (don't catch errors) create_kwargs = {} # we must always add a key, so if not given we use a shortened md5 hash. There is a (small) # chance this is not unique but it should usually not be a problem. val = prot.pop("key", "Spawned-{}".format( hashlib.md5(str(time.time())).hexdigest()[:6])) create_kwargs["db_key"] = init_spawn_value(val, str) val = prot.pop("location", None) create_kwargs["db_location"] = init_spawn_value(val, value_to_obj) val = prot.pop("home", settings.DEFAULT_HOME) create_kwargs["db_home"] = init_spawn_value(val, value_to_obj) val = prot.pop("destination", None) create_kwargs["db_destination"] = init_spawn_value(val, value_to_obj) val = prot.pop("typeclass", settings.BASE_OBJECT_TYPECLASS) create_kwargs["db_typeclass_path"] = init_spawn_value(val, str) # extract calls to handlers val = prot.pop("permissions", []) permission_string = init_spawn_value(val, make_iter) val = prot.pop("locks", "") lock_string = init_spawn_value(val, str) val = prot.pop("aliases", []) alias_string = init_spawn_value(val, make_iter) val = prot.pop("tags", []) tags = [] for (tag, category, data) in tags: tags.append((init_spawn_value(val, str), category, data)) prototype_key = prototype.get('prototype_key', None) if prototype_key: # we make sure to add a tag identifying which prototype created this object tags.append((prototype_key, _PROTOTYPE_TAG_CATEGORY)) val = prot.pop("exec", "") execs = init_spawn_value(val, make_iter) # extract ndb assignments nattributes = dict((key.split("_", 1)[1], init_spawn_value(val, value_to_obj)) for key, val in prot.items() if key.startswith("ndb_")) # the rest are attribute tuples (attrname, value, category, locks) val = make_iter(prot.pop("attrs", [])) attributes = [] for (attrname, value, category, locks) in val: attributes.append((attrname, init_spawn_value(val), category, locks)) simple_attributes = [] for key, value in ((key, value) for key, value in prot.items() if not (key.startswith("ndb_"))): # we don't support categories, nor locks for simple attributes if key in _PROTOTYPE_META_NAMES: continue else: simple_attributes.append( (key, init_spawn_value(value, value_to_obj_or_any), None, None)) attributes = attributes + simple_attributes attributes = [tup for tup in attributes if not tup[0] in _NON_CREATE_KWARGS] # pack for call into _batch_create_object objsparams.append((create_kwargs, permission_string, lock_string, alias_string, nattributes, attributes, tags, execs)) if kwargs.get("only_validate"): return objsparams return batch_create_object(*objsparams)
def batch_update_objects_with_prototype(prototype, diff=None, objects=None, exact=False): """ Update existing objects with the latest version of the prototype. Args: prototype (str or dict): Either the `prototype_key` to use or the prototype dict itself. diff (dict, optional): This a diff structure that describes how to update the protototype. If not given this will be constructed from the first object found. objects (list, optional): List of objects to update. If not given, query for these objects using the prototype's `prototype_key`. exact (bool, optional): By default (`False`), keys not explicitly in the prototype will not be applied to the object, but will be retained as-is. This is usually what is expected - for example, one usually do not want to remove the object's location even if it's not set in the prototype. With `exact=True`, all un-specified properties of the objects will be removed if they exist. This will lead to a more accurate 1:1 correlation between the object and the prototype but is usually impractical. Returns: changed (int): The number of objects that had changes applied to them. """ prototype = protlib.homogenize_prototype(prototype) if isinstance(prototype, str): new_prototype = protlib.search_prototype(prototype) else: new_prototype = prototype prototype_key = new_prototype["prototype_key"] if not objects: objects = ObjectDB.objects.get_by_tag(prototype_key, category=PROTOTYPE_TAG_CATEGORY) if not objects: return 0 if not diff: diff, _ = prototype_diff_from_object(new_prototype, objects[0]) # make sure the diff is flattened diff = flatten_diff(diff) changed = 0 for obj in objects: do_save = False old_prot_key = obj.tags.get(category=PROTOTYPE_TAG_CATEGORY, return_list=True) old_prot_key = old_prot_key[0] if old_prot_key else None try: for key, directive in diff.items(): if key not in new_prototype and not exact: # we don't update the object if the prototype does not actually # contain the key (the diff will report REMOVE but we ignore it # since exact=False) continue if directive in ("UPDATE", "REPLACE"): if key in _PROTOTYPE_META_NAMES: # prototype meta keys are not stored on-object continue val = new_prototype[key] do_save = True if key == "key": obj.db_key = init_spawn_value(val, str) elif key == "typeclass": obj.db_typeclass_path = init_spawn_value(val, str) elif key == "location": obj.db_location = init_spawn_value(val, value_to_obj) elif key == "home": obj.db_home = init_spawn_value(val, value_to_obj) elif key == "destination": obj.db_destination = init_spawn_value(val, value_to_obj) elif key == "locks": if directive == "REPLACE": obj.locks.clear() obj.locks.add(init_spawn_value(val, str)) elif key == "permissions": if directive == "REPLACE": obj.permissions.clear() obj.permissions.batch_add(*(init_spawn_value(perm, str) for perm in val)) elif key == "aliases": if directive == "REPLACE": obj.aliases.clear() obj.aliases.batch_add(*(init_spawn_value(alias, str) for alias in val)) elif key == "tags": if directive == "REPLACE": obj.tags.clear() obj.tags.batch_add( *( (init_spawn_value(ttag, str), tcategory, tdata) for ttag, tcategory, tdata in val ) ) elif key == "attrs": if directive == "REPLACE": obj.attributes.clear() obj.attributes.batch_add( *( ( init_spawn_value(akey, str), init_spawn_value(aval, value_to_obj), acategory, alocks, ) for akey, aval, acategory, alocks in val ) ) elif key == "exec": # we don't auto-rerun exec statements, it would be huge security risk! pass else: obj.attributes.add(key, init_spawn_value(val, value_to_obj)) elif directive == "REMOVE": do_save = True if key == "key": obj.db_key = "" elif key == "typeclass": # fall back to default obj.db_typeclass_path = settings.BASE_OBJECT_TYPECLASS elif key == "location": obj.db_location = None elif key == "home": obj.db_home = None elif key == "destination": obj.db_destination = None elif key == "locks": obj.locks.clear() elif key == "permissions": obj.permissions.clear() elif key == "aliases": obj.aliases.clear() elif key == "tags": obj.tags.clear() elif key == "attrs": obj.attributes.clear() elif key == "exec": # we don't auto-rerun exec statements, it would be huge security risk! pass else: obj.attributes.remove(key) except Exception: logger.log_trace(f"Failed to apply prototype '{prototype_key}' to {obj}.") finally: # we must always make sure to re-add the prototype tag obj.tags.clear(category=PROTOTYPE_TAG_CATEGORY) obj.tags.add(prototype_key, category=PROTOTYPE_TAG_CATEGORY) if do_save: changed += 1 obj.save() return changed