Exemplo n.º 1
0
 def __init__(self, address="http://localhost:12000"):
     '''
     address: includes port. e.g. http://localhost:9000
     sim: identifier name for the simulator using this store
     '''
     self.encoder = CADISEncoder()
     if not address.endswith('/'):
         address += '/'
     self.address = address
Exemplo n.º 2
0
 def __init__(self):
     '''
     Constructor
     '''
     self.store = {}
     self.address = "http://localhost:12000/"
     for t in schema.sets:
         self.store[t] = {}
     self.encoder = CADISEncoder()
Exemplo n.º 3
0
    def __init__(self, store=None):
        '''
        Constructor
        '''
        self.app = None
        self.timer = None
        self.interval = None
        self.track_changes = False
        self.step = 0
        self.curtime = time.time()
        self.__Logger = logger

        # Local storage for thread
        self.tlocal = None

        # Stores objects from store
        self.storebuffer = {}

        # Stores new objects since last pull
        self.new_storebuffer = {}
        # Stores modified objects since last pull
        self.mod_storebuffer = {}
        # Store removed objects since last pull
        self.del_storebuffer = {}

        # Types that are being observed (read) by this application
        self.observed = set()
        # Stores the types that can be updated (read/write) by app
        self.updated = set()
        # Stores the types that can be produced by app
        self.produced = set()
        # Stores the types that can be retrieved by app
        self.newlyproduced = {}

        # Properties that changed during this iteration
        self.changedproperties = {}

        # List of objects to be pushed to the store (no subsets)
        self.pushlist = {}

        # List of objects marked for deletion
        self.deletelist = {}

        # Holds a k,v storage of foreign keys (e.g. name -> id)
        self.fkdict = {}

        # Disables fetching of subsets
        self.subset_disable = set()

        self.timer = None
        self.step = 0
        self.thread = None
        self.encoder = CADISEncoder()
        if store:
            logger.debug("%s received store %s", LOG_HEADER, store)
            Frame.Store = store
Exemplo n.º 4
0
 def __init__(self, address="http://localhost:12000"):
     '''
     address: includes port. e.g. http://localhost:9000
     sim: identifier name for the simulator using this store
     '''
     self.encoder = CADISEncoder()
     if not address.endswith('/'):
         address += '/'
     self.address = address
Exemplo n.º 5
0
 def __init__(self):
     '''
     Constructor
     '''
     self.store = {}
     self.address = "http://localhost:12000/"
     for t in schema.sets:
         self.store[t] = {}
     self.encoder = CADISEncoder()
Exemplo n.º 6
0
    def __init__(self, store=None):
        '''
        Constructor
        '''
        self.app = None
        self.timer = None
        self.interval = None
        self.track_changes = False
        self.step = 0
        self.curtime = time.time()
        self.__Logger = logger

        # Local storage for thread
        self.tlocal = None

        # Stores objects from store
        self.storebuffer = {}

        # Stores new objects since last pull
        self.new_storebuffer = {}
        # Stores modified objects since last pull
        self.mod_storebuffer = {}
        # Store removed objects since last pull
        self.del_storebuffer = {}

        # Types that are being observed (read) by this application
        self.observed = set()
        # Stores the types that can be updated (read/write) by app
        self.updated = set()
        # Stores the types that can be produced by app
        self.produced = set()
        # Stores the types that can be retrieved by app
        self.newlyproduced = {}

        # Properties that changed during this iteration
        self.changedproperties = {}

        # List of objects to be pushed to the store (no subsets)
        self.pushlist = {}

        # List of objects marked for deletion
        self.deletelist = {}

        # Holds a k,v storage of foreign keys (e.g. name -> id)
        self.fkdict = {}

        # Disables fetching of subsets
        self.subset_disable = set()

        self.timer = None
        self.step = 0
        self.thread = None
        self.encoder = CADISEncoder()
        if store:
            logger.debug("%s received store %s", LOG_HEADER, store)
            Frame.Store = store
Exemplo n.º 7
0
class RemoteStore(IStore):
    '''
    classdocs
    '''
    app = None
    store = {}
    address = ""

    def __init__(self):
        '''
        Constructor
        '''
        self.store = {}
        self.address = "http://localhost:12000/"
        for t in schema.sets:
            self.store[t] = {}
        self.encoder = CADISEncoder()

    def insert(self, obj, sim=None):
        msg = self.encoder.encode(obj)
        req = urllib2.Request(self.address + obj._FULLNAME + '/')
        req.add_header('Content-Type', 'application/json')

        response = urllib2.urlopen(req, msg)
        return response

    def get(self, typeObj):
        jsonlist = json.load(
            urllib2.urlopen(self.address + typeObj._FULLNAME + "/"))
        objlist = []
        for data in jsonlist:
            obj = typeObj.__new__(typeObj)
            for dim in obj._dimensions:
                prop = getattr(obj, dim._name)
                if hasattr(prop, "__decode__"):
                    prop = prop.__decode__(data[dim._name])
                else:
                    prop = data[dim._name]
                setattr(obj, dim._name, prop)
            obj.ID = UUID(data["ID"])
            objlist.append(obj)
        return objlist

    def delete(self, typeObj, obj):
        pass

    def close(self):
        return
Exemplo n.º 8
0
class RemoteStore(IStore):
    '''
    classdocs
    '''
    app = None
    store = {}
    address = ""

    def __init__(self):
        '''
        Constructor
        '''
        self.store = {}
        self.address = "http://localhost:12000/"
        for t in schema.sets:
            self.store[t] = {}
        self.encoder = CADISEncoder()

    def insert(self, obj, sim = None):
        msg = self.encoder.encode(obj)
        req = urllib2.Request(self.address + obj._FULLNAME + '/')
        req.add_header('Content-Type', 'application/json')

        response = urllib2.urlopen(req, msg)
        return response

    def get(self, typeObj):
        jsonlist = json.load(urllib2.urlopen(self.address + typeObj._FULLNAME + "/"))
        objlist = []
        for data in jsonlist:
            obj = typeObj.__new__(typeObj)
            for dim in obj._dimensions:
                prop = getattr(obj, dim._name)
                if hasattr(prop, "__decode__"):
                    prop = prop.__decode__(data[dim._name])
                else:
                    prop = data[dim._name]
                setattr(obj, dim._name, prop)
            obj.ID = UUID(data["ID"])
            objlist.append(obj)
        return objlist

    def delete(self, typeObj, obj):
        pass

    def close(self):
        return
Exemplo n.º 9
0
class PythonRemoteStore(IStore):
    __Logger = logging.getLogger(__name__)
    def __init__(self, address="http://localhost:12000"):
        '''
        address: includes port. e.g. http://localhost:9000
        sim: identifier name for the simulator using this store
        '''
        self.encoder = CADISEncoder()
        if not address.endswith('/'):
            address += '/'
        self.address = address

    def insert(self, obj, sim):
        jsonobj = self.encoder.encode(obj)
        response = requests.put("%s%s/%s" % (self.base_address, obj._FULLNAME, obj.ID), data={'obj' : jsonobj }, headers={'Content-Type':'application/json', 'InsertType':'Single'})
        return response

    def insert_all(self, t, list_obj, sim):
        objs = self.encoder.encode(list_obj)
        response = requests.put(self.base_address + t._FULLNAME, data={ 'insert_list' : objs }, headers={'InsertType':'Multiple'}, stream=True)
        return response

    def get(self, typeObj):
        resp = requests.get(self.base_address + typeObj._FULLNAME)
        jsonlist = json.loads(resp.text)
        objlist = []
        for data in jsonlist:
            # obj = typeObj.__new__(typeObj)
            obj = CADIS()
            obj.__class__ = typeObj
            for dim in obj._dimensions:
                prop = getattr(obj, dim._name)
                if hasattr(prop, "__decode__"):
                    prop = prop.__decode__(data[dim._name])
                else:
                    prop = data[dim._name]
                setattr(obj, dim._name, prop)
            obj.ID = UUID(data["ID"])
            objlist.append(obj)
        return objlist

    def register(self, sim):
        self.base_address = self.address + sim + '/'
        resp = requests.put(self.base_address[:-1])
        self.sim = sim
        return resp

    def update_all(self, pushlist, sim):
        for t in pushlist:
            if len(pushlist[t]) > 0:
                tmp = pushlist[t]
                updates = self.encoder.encode({str(k):v for k, v in tmp.items()})
                resp = requests.post(self.base_address + t._FULLNAME, data={'update_dict' : updates})
            pushlist[t] = {}

    def create_obj(self, typeObj, data):
        try:
            obj = typeObj.__new__(typeObj)
            for dim in obj._dimensions:
                prop = getattr(obj, dim._name)
                if hasattr(prop, "__decode__"):
                    prop = prop.__decode__(data[dim._name])
                else:
                    prop = data[dim._name]
                setattr(obj, dim._name, prop)
            obj.ID = UUID(data["ID"])
        except Exception, e:
            self.__Logger.exception("Failed to create object from data %s", data)
        return obj
Exemplo n.º 10
0
 def __init__(self):
     '''
     Constructor
     '''
     self.address = "http://localhost:12000/"
     self.encoder = CADISEncoder()
Exemplo n.º 11
0
class Frame(object):
    '''
    classdocs
    '''

    Store = None

    def __init__(self, store=None, process=False, settings=None):
        '''
        Constructor
        '''
        self.app = None
        self.timer = None
        self.interval = None
        self.track_changes = False
        self.step = 0
        self.curtime = time.time()
        self.__Logger = logger
        self.process = process
        self.settings = settings

        # Local storage for thread
        self.tlocal = None

        # Stores objects from store
        self.storebuffer = {}

        # Stores new objects since last pull
        self.new_storebuffer = {}
        # Stores modified objects since last pull
        self.mod_storebuffer = {}
        # Store removed objects since last pull
        self.del_storebuffer = {}

        # Types that are being observed (read) by this application
        self.observed = set()
        # Stores the types that can be updated (read/write) by app
        self.updated = set()
        # Stores the types that can be produced by app
        self.produced = set()
        # Stores the types that can be retrieved by app
        self.newlyproduced = {}
        # Store the types "tracked" by the app (i.e. creation and deletion only)
        self.tracked = {}

        # Properties that changed during this iteration
        self.changedproperties = {}

        # List of objects to be pushed to the store (no subsets)
        self.pushlist = {}

        # List of objects marked for deletion
        self.deletelist = {}

        # Holds a k,v storage of foreign keys (e.g. name -> id)
        self.fkdict = {}

        # Disables fetching of subsets
        self.subset_disable = set()

        # Keeps track of object ids received for subset queries that are not in buffer yet
        self.orphan_objids = {}

        self.timer = None
        self.step = 0
        self.thread = None
        self.encoder = CADISEncoder()
        if store:
            logger.debug("%s received store %s", LOG_HEADER, store)
            Frame.Store = store

    ######################################################
    ## Application -> Frame Functions
    ######################################################
    def add(self, obj):
        t = obj.__class__
        if t in self.storebuffer:
            # logger.debug("%s Creating new object %s.%s", LOG_HEADER, obj.__class__, obj._primarykey)
            if obj._primarykey == None:
                obj._primarykey = uuid.uuid4()
            obj._frame = self
            self.newlyproduced[t][obj._primarykey] = obj
            self.storebuffer[t][obj._primarykey] = obj
            # If we removed then readded in the same tick, make sure we don't send the remove anymore
            if t in self.deletelist and obj._primarykey in self.deletelist[t]:
                del self.deletelist[t][obj._primarykey]
        else:
            logger.error("%s Object not in dictionary: %s", LOG_HEADER,
                         obj.__class__)

    def delete(self, t, oid):
        if oid in self.storebuffer[t]:
            o = self.storebuffer[t][oid]
            self.deletelist[t][o._primarykey] = o
            del self.storebuffer[t][oid]
            # If we added and removed in the same tick, make sure we don't send the add
            if t in self.newlyproduced and o._primarykey in self.newlyproduced[
                    t]:
                del self.newlyproduced[t][o._primarykey]

        else:
            logger.warn("could not find object %s in storebuffer")
        return True

    def get(self, t, primkey=None):
        self.track_changes = False
        try:
            if t not in self.storebuffer:
                self.__Logger.error(
                    "Could not find type %s in cache. Did you remember to add it as a gettter, setter, or producer in the simulator?"
                )
                sys.exit(0)
            if primkey != None:
                obj = self.storebuffer[t][primkey]
                if hasattr(t, '_foreignkeys'):
                    self._resolve_fk(obj, t)
                return obj
            else:
                res = []
                for obj in self.storebuffer[t].values():
                    res.append(self.get(t, obj._primarykey))
                return res
        except:
            self.__Logger.exception("Uncaught exception in Frame.Get")
            raise
        finally:
            self.track_changes = True

    def new(self, t):
        return self.new_storebuffer[t].values()

    def deleted(self, t):
        return self.del_storebuffer[t].values()

    def changed(self, t):
        return self.mod_storebuffer[t].values()

    def disable_subset(self, t):
        self.subset_disable.add(t)

    def enable_subset(self, t):
        if t in self.subset_disable:
            self.subset_disable.remove(t)

    def attach(self, app):
        self.app = app
        self.process_declarations(app)
        #self.app.initialize()

    def go(self, cmd_dict, timer=None):
        self.cmds = cmd_dict
        self.timer = timer
        self.runner = TimerThread(self, Frame.Store, cmd_dict, timer)
        if self.process:
            self.thread = Process(target=self.runner.run)
        else:
            self.thread = Thread(target=self.runner.run)
        self.thread.start()

    def join(self):
        self.thread.join()

    ######################################################
    ## Core Functions
    ######################################################
    def execute_Frame(self):
        try:
            self.pull()
            self.track_changes = True
            self.app.update()
            self.track_changes = False
            self.push()
            self.step += 1
            self.curtime = time.time()
        except:
            self.cmds["SimulatorPaused"] = True
            logger.exception("[%s] uncaught exception: ", self.app.__module__)

    def initialize_app(self):
        self.app.initialize()
        # Push initial objects the application has added.
        self.push()
        self._update_shared_status("Pushed")
        logger.warn("Application %s finished pushing", self.app._appname)
        it = 0
        done = False
        while (it < 300 and done == False
               and self.cmds["SimulatorShutdown"] == False):
            done = True
            for status in self._get_all_shared_statuses().values():
                if status != "Pushed":
                    done = False
                    break
            it += 1
            time.sleep(0.1)
        self.pull()

    def process_declarations(self, app):
        self.produced = app._producer
        self.updated = app._gettersetter
        self.tracked_only = set()

        self.observed = set()
        self.iterate_types = []
        if app._getter:
            self.observed = self.observed.union(app._getter)
        if app._gettersetter:
            self.observed = self.observed.union(app._gettersetter)
        if app._tracker:
            self.tracked_only = app._tracker.difference(self.observed)
            self.observed = self.observed.union(app._tracker)

        # Check for "parents" of subsets, they need to be fetched too
        for t in copy(self.observed):
            if t in schema_subsets:
                pt = setsof[t]
                self.observed.add(pt)

        for t in self.observed.union(self.produced).union(self.updated):
            # self.changedproperties[t] = {}
            self.storebuffer[t] = {}
            self.new_storebuffer[t] = {}
            self.mod_storebuffer[t] = {}
            self.del_storebuffer[t] = {}
            if hasattr(t, "_foreignkeys"):
                for propname, cls in t._foreignkeys.items():
                    fname = getattr(t, propname)._foreignprop._name
                    if not cls in self.fkdict:
                        self.fkdict[cls] = {}
                    if not fname in self.fkdict[cls]:
                        self.fkdict[cls][fname] = {}
            logger.debug("%s store buffer for type %s", LOG_HEADER, t)

        # When iterating for pull, we should request, in order:
        # (1) set items
        # (2) permutations
        # (3) subsets

        setitems = self.observed.intersection(schema_sets)
        permutations = self.observed.intersection(schema_permutationsets)
        subsets = self.observed.intersection(schema_subsets)
        for t in subsets:
            self.orphan_objids[t] = set()

        self.iterate_types.extend(setitems)
        self.iterate_types.extend(permutations)
        self.iterate_types.extend(subsets)

        # prepare structures for storing and pushing new, update, and deletion of objects
        for t in self.produced:
            self.newlyproduced[t] = {}
        for t in self.updated:
            self.deletelist[t] = {}
            self.pushlist[t] = {}

        if Frame.Store == None:
            logger.debug("%s creating new store", LOG_HEADER)
            if USE_REMOTE_STORE:
                Frame.Store = RemoteStore()
            else:
                Frame.Store = SimpleStore()
        setattr(self.app, "_appname", self.app.__class__.__name__)
        Frame.Store.register(self.app._appname)

    def stop(self):
        self.app.shutdown()

    ######################################################
    ## Frame -> Store Functions
    ######################################################
    @instrument
    def pull(self):
        tmpbuffer = {}
        for t in self.iterate_types:
            if t in self.subset_disable:
                continue

            self.new_storebuffer[t] = {}
            self.mod_storebuffer[t] = {}
            self.del_storebuffer[t] = {}

            # optimization: retrieve only objects that changed since last pull
            if hasattr(Frame.Store, "getupdated"):
                if t in self.tracked_only:
                    (new, mod,
                     deleted) = Frame.Store.getupdated(t,
                                                       self.app._appname,
                                                       tracked_only=True)
                else:
                    (new, mod,
                     deleted) = Frame.Store.getupdated(t, self.app._appname)
                if t in schema_subsets:
                    # Parent type of subset 't'
                    pt = setsof[t]

                    # Iterate for orphan keys from last pull
                    notfound = set()
                    for key in self.orphan_objids[t]:
                        if key in self.storebuffer[pt]:
                            o = copy(self.storebuffer[pt][key])
                            o.__class__ = t
                            self.storebuffer[t][key] = o
                            self.new_storebuffer[t][key] = o
                        else:
                            notfound.add(key)
                            self.__Logger.error(
                                "missing key %s for new object in subset list",
                                key)
                    self.orphan_objids[t] = set().union(notfound)

                    for key in new:
                        # If the key is not in the store's buffer, its likely the object was created between
                        # the get for parent type and the current subset query. This is expected, just keep a reference
                        # to check for again on next pull.
                        if key in self.storebuffer[pt]:
                            o = copy(self.storebuffer[pt][key])
                            o.__class__ = t
                            self.storebuffer[t][key] = o
                            self.new_storebuffer[t][key] = o
                        else:
                            self.orphan_objids[t].add(key)
                    for key in mod:
                        o = copy(self.storebuffer[pt][key])
                        o.__class__ = t
                        self.storebuffer[t][key] = o
                        self.mod_storebuffer[t][key] = o
                    for key in deleted:
                        if key in self.storebuffer[pt]:
                            o = copy(self.storebuffer[pt][key])
                        elif key in self.del_storebuffer[pt]:
                            o = copy(self.del_storebuffer[pt][key])
                        else:
                            self.__Logger.warn(
                                "object %s was deleted, could find find a reference to give to application.",
                                key)
                            o = CADIS()
                            o.ID = key
                        o.__class__ = t
                        del self.storebuffer[t][key]
                        self.del_storebuffer[t][key] = o
                else:
                    for o in new:
                        self.storebuffer[t][o._primarykey] = o
                        self.new_storebuffer[t][o._primarykey] = o
                        if o.__class__ in self.fkdict:
                            self._fkobj(o)
                    for o in mod:
                        self.storebuffer[t][o._primarykey] = o
                        self.mod_storebuffer[t][o._primarykey] = o
                        if o.__class__ in self.fkdict:
                            self._fkobj(o)
                    for key in deleted:
                        if key in self.storebuffer[t]:
                            o = self.storebuffer[t][key]
                            self.del_storebuffer[t][key] = o
                            del self.storebuffer[t][key]
                        if o.__class__ in self.fkdict:
                            self._delfkobj(o)

            else:
                tmpbuffer[t] = {}
                for o in Frame.Store.get(t):
                    tmpbuffer[t][o._primarykey] = o

                # TODO: Remove when Store does this
                # Added objects since last pull
                for o in tmpbuffer[t].values():
                    if o._primarykey not in self.storebuffer[t]:
                        # logger.debug("%s Found new object: %s", LOG_HEADER, o)
                        self.new_storebuffer[t][o._primarykey] = o
                    else:
                        # check if updated
                        orig = self.encoder.encode(
                            self.storebuffer[t][o._primarykey])
                        new = self.encoder.encode(o)
                        if orig != new:
                            self.mod_storebuffer[t][o._primarykey] = o

                # Deleted objects since last pull
                for o in self.storebuffer[t].values():
                    if o._primarykey not in tmpbuffer[t]:
                        self.del_storebuffer[t][o._primarykey] = o

                self.storebuffer[t] = tmpbuffer[t]

    @instrument
    def push(self):
        cleartypes = set()
        for t in self.newlyproduced:
            if len(self.newlyproduced[t]) > 0:
                if hasattr(Frame.Store, "insert_all"):
                    Frame.Store.insert_all(t, self.newlyproduced[t].values(),
                                           self.app._appname)
                else:
                    for o in self.newlyproduced[t].values():
                        Frame.Store.insert(o, self.app._appname)
            if t not in self.observed:
                cleartypes.add(t)
            self.newlyproduced[t] = {}

        for t in self.deletelist:
            for o in self.deletelist[t].values():
                Frame.Store.delete(t, o._primarykey, self.app._appname)
                if t in self.pushlist and o._primarykey in self.pushlist[t]:
                    del self.pushlist[t][o._primarykey]
            self.deletelist[t] = {}

        Frame.Store.update_all(self.pushlist, self.app._appname)
        for t in cleartypes:
            self.storebuffer[t] = {}

    ######################################################
    ## Utility Functions
    ######################################################
    def buffersize(self):
        size = 0
        nobjects = 0
        for t in self.storebuffer:
            for o in self.storebuffer[t].values():
                nobjects += 1
                for prop in o._dimensions:
                    size += sys.getsizeof(prop)
        return (nobjects, size)

    def _fkobj(self, o):
        for propname in self.fkdict[o.__class__].keys():
            propvalue = getattr(o, propname)
            self.fkdict[o.__class__][propname][propvalue] = o.ID

    def _delfkobj(self, o):
        for propname in self.fkdict[o.__class__].keys():
            propvalue = getattr(o, propname)
            if propvalue in self.fkdict[o.__class__][propname]:
                del self.fkdict[o.__class__][propname][propvalue]

    def name2class(self, typeName):
        for t in self.storebuffer:
            if t.__name__ == typeName:
                return t

    def count(self, typeObj):
        if typeObj in self.storebuffer:
            return len(self.storebuffer[typeObj])
        else:
            return -1

    def _update_shared_status(self, newstatus):
        self.cmds["APP_" + self.app._appname] = newstatus

    def _get_all_shared_statuses(self):
        res = {}
        for k, v in self.cmds.items():
            if k.startswith("APP_"):
                res[k] = v
        return res

    def findproperty(self, t, propname, value):
        for o in self.storebuffer[t].values():
            if getattr(o, propname) == value:
                return o
        return None

    def set_property(self, t, o, v, n):
        # if not a gettersetter for this type, return
        if t not in self.updated:
            return

        # Newly produced items will be pushed entirely. Skip...
        if not o._primarykey:
            return

        if t in self.newlyproduced and o._primarykey in self.newlyproduced[t]:
            # logger.debug("[%s] Object ID %s in newly produced")
            return

        # Object not tracked by store yet. Ignore...
        if o._primarykey not in self.storebuffer[t]:
            # logger.debug("[%s] Object ID %s not being tracked yet")
            return

        if o._primarykey not in self.pushlist[t]:
            self.pushlist[t][o._primarykey] = {}

        # logger.debug("[%s] Object ID %s property %s being set to %s")
        # Save the property update
        self.pushlist[t][o._primarykey][n] = v
        # logger.debug("property %s of object %s (ID %s) set to %s", n, o, o._primarykey, v)

    def _resolve_fk(self, obj, t):
        for propname, cls in t._foreignkeys.items():
            propvalue = getattr(obj, propname)
            fname = getattr(t, propname)._foreignprop._name
            if propvalue in self.fkdict[cls][fname]:
                primkey = self.fkdict[cls][fname][propvalue]
            else:
                # logger.error("Could not find property value in foreign key dictionary.")
                return None
            if primkey in self.storebuffer[cls]:
                newobj = self.storebuffer[cls][primkey]
                if hasattr(cls, '_foreignkeys'):
                    self._resolve_fk(newobj, cls)
                setattr(obj, propname, newobj)
            # Look up by name, just in case
            else:
                fname = getattr(obj.__class__, propname)._foreignprop._name
                for o in self.storebuffer[cls].values():
                    if propvalue == getattr(o, fname):
                        setattr(obj, propname, o)

    def __deepcopy__(self, memo):
        return self
Exemplo n.º 12
0
class Frame(object):
    '''
    classdocs
    '''

    Store = None
    def __init__(self, store=None):
        '''
        Constructor
        '''
        self.app = None
        self.timer = None
        self.interval = None
        self.track_changes = False
        self.step = 0
        self.curtime = time.time()
        self.__Logger = logger

        # Local storage for thread
        self.tlocal = None

        # Stores objects from store
        self.storebuffer = {}

        # Stores new objects since last pull
        self.new_storebuffer = {}
        # Stores modified objects since last pull
        self.mod_storebuffer = {}
        # Store removed objects since last pull
        self.del_storebuffer = {}

        # Types that are being observed (read) by this application
        self.observed = set()
        # Stores the types that can be updated (read/write) by app
        self.updated = set()
        # Stores the types that can be produced by app
        self.produced = set()
        # Stores the types that can be retrieved by app
        self.newlyproduced = {}

        # Properties that changed during this iteration
        self.changedproperties = {}

        # List of objects to be pushed to the store (no subsets)
        self.pushlist = {}

        # List of objects marked for deletion
        self.deletelist = {}

        # Holds a k,v storage of foreign keys (e.g. name -> id)
        self.fkdict = {}

        # Disables fetching of subsets
        self.subset_disable = set()

        self.timer = None
        self.step = 0
        self.thread = None
        self.encoder = CADISEncoder()
        if store:
            logger.debug("%s received store %s", LOG_HEADER, store)
            Frame.Store = store

    def attach(self, app):
        self.app = app
        self.process_declarations(app)
        self.app.initialize()

    def go(self):
        #self.timer = Timer(1.0, self.execute_Frame)
        #self.timer.start()
        self.thread = TimerThread(self)
        self.thread.start()
        #thread.join()
        #self.stop()
    def join(self):
        self.thread.join()

    def stop(self):
        if self.timer:
            self.timer.cancel()
        sys.exit(0)

    def disable_subset(self, t):
        self.subset_disable.add(t)

    def enable_subset(self, t):
        if t in self.subset_disable:
            self.subset_disable.remove(t)

    def process_declarations(self, app):
        self.produced = app._producer
        self.updated = app._gettersetter
        self.observed = set()
        if app._getter:
            self.observed = self.observed.union(app._getter)
        if app._gettersetter:
            self.observed = self.observed.union(app._gettersetter)

        for t in self.observed.union(self.produced).union(self.updated):
            #self.changedproperties[t] = {}
            self.storebuffer[t] = {}
            self.new_storebuffer[t] = {}
            self.mod_storebuffer[t] = {}
            self.del_storebuffer[t] = {}
            if hasattr(t, "_foreignkeys"):
                for propname, cls in t._foreignkeys.items():
                    fname = getattr(t, propname)._foreignprop._name
                    if not cls in self.fkdict:
                        self.fkdict[cls] = {}
                    if not fname in self.fkdict[cls]:
                        self.fkdict[cls][fname] = {}
            logger.debug("%s store buffer for type %s", LOG_HEADER, t)

        for t in self.produced:
            self.pushlist[t] = {}
            self.deletelist[t] = {}
            self.newlyproduced[t] = {}

        if Frame.Store == None:
            logger.debug("%s creating new store", LOG_HEADER)
            if USE_REMOTE_STORE:
                Frame.Store = RemoteStore()
            else:
                Frame.Store = SimpleStore()

        Frame.Store.register(self)

    def execute_Frame(self):
        try:
            self.pull()
            self.track_changes = True
            self.app.update()
            self.track_changes = False
            self.prepare_push()
            self.push()
            if self.timer:
                self.timer = Timer(1.0, self.execute_Frame)
                self.timer.start()
            self.step += 1
            self.curtime = time.time()
        except:
            logger.exception("uncaught exception: ")

    def pull(self):
        tmpbuffer = {}
        for t in self.observed.symmetric_difference(self.subset_disable):
            self.new_storebuffer[t] = {}
            self.mod_storebuffer[t] = {}
            self.del_storebuffer[t] = {}

            # if type is subset, it will always recalculate and return as new / updated
            # TODO: Allow caching of subsets to determine new / updated / deleted
            if hasattr(Frame.Store, "updated"):
                (new, mod, deleted) = Frame.Store.updated(t, self)
                for o in new:
                    self.storebuffer[t][o._primarykey] = o
                    self.new_storebuffer[t][o._primarykey] = o
                    if o.__class__ in self.fkdict:
                        for propname in self.fkdict[o.__class__].keys():
                            propvalue = getattr(o, propname)
                            self.fkdict[o.__class__][propname][propvalue] = o.ID
                for o in mod:
                    self.storebuffer[t][o._primarykey] = o
                    self.mod_storebuffer[t][o._primarykey] = o
                    if o.__class__ in self.fkdict:
                        for propname in self.fkdict[o.__class__].keys():
                            propvalue = getattr(o, propname)
                            self.fkdict[o.__class__][propname][propvalue] = o.ID
                for o in deleted:
                    if o._primarykey in self.storebuffer[t]:
                        del self.storebuffer[t][o._primarykey]
                        self.del_storebuffer[t][o._primarykey] = o
                    if o.__class__ in self.fkdict:
                        for propname in self.fkdict[o.__class__].keys():
                            propvalue = getattr(o, propname)
                            if propvalue in self.fkdict[o.__class__][propname]:
                                del self.fkdict[o.__class__][propname][propvalue]
            else:
                tmpbuffer[t] = {}
                for o in Frame.Store.get(t):
                    tmpbuffer[t][o._primarykey] = o

                #TODO: Remove when Store does this
                # Added objects since last pull
                for o in tmpbuffer[t].values():
                    if o._primarykey not in self.storebuffer[t]:
                        #logger.debug("%s Found new object: %s", LOG_HEADER, o)
                        self.new_storebuffer[t][o._primarykey] = o
                    else:
                        # check if updated
                        orig = self.encoder.encode(self.storebuffer[t][o._primarykey])
                        new = self.encoder.encode(o)
                        if orig != new:
                            self.mod_storebuffer[t][o._primarykey] = o

                # Deleted objects since last pull
                for o in self.storebuffer[t].values():
                    if o._primarykey not in tmpbuffer[t]:
                        self.del_storebuffer[t][o._primarykey] = o

                self.storebuffer[t] = tmpbuffer[t]

    def prepare_push(self):
        #for t in self.changedproperties:
        #    push_key = None
        #    push_obj = None
        #    if t in subsets:
        #        push_key = setsof[t]
        #
        #    elif t in sets:
        #        push_key = t
        #        for o in self.changedproperties[t]:
        #            self.pushlist[t][o._primarykey] = o
        pass

    def push(self):
        for t in self.pushlist:
            for primkey in self.pushlist[t].keys():
                if primkey in self.newlyproduced[t]:
                    continue
                elif primkey in self.storebuffer[t]:
                    Frame.Store.update(t, primkey, self, self.pushlist[t][primkey])
                else:
                    logger.error("[%s] Missing object in store buffer. Object ID %s,  Pushlist: %s", self.app, primkey, self.pushlist[t])
            self.pushlist[t] = {}

        for t in self.produced:
            if t in self.newlyproduced:
                for o in self.newlyproduced[t].values():
                    #logger.debug("[%s] Adding obj %s (ID %s) to store", self.app, o, o._primarykey)
                    Frame.Store.insert(o, self)
                self.newlyproduced[t] = {}

        for t in self.deletelist:
            for o in self.deletelist[t].values():
                Frame.Store.delete(t, o)

    def set_property(self, t, o, v, n):
        # Newly produced items will be pushed entirely. Skip...
        if not o._primarykey:
            return

        if t in self.newlyproduced and o._primarykey in self.newlyproduced[t]:
            #logger.debug("[%s] Object ID %s in newly produced")
            return

        # Object not tracked by store yet. Ignore...
        if o._primarykey not in self.storebuffer[t]:
            #logger.debug("[%s] Object ID %s not being tracked yet")
            return

        if o._primarykey not in self.pushlist[t]:
            self.pushlist[t][o._primarykey] = {}

        #logger.debug("[%s] Object ID %s property %s being set to %s")
        # Save the property update
        self.pushlist[t][o._primarykey][n] = v
        #logger.debug("property %s of object %s (ID %s) set to %s", n, o, o._primarykey, v)

    def add(self, obj):
        if obj.__class__ in self.storebuffer:
            #logger.debug("%s Creating new object %s.%s", LOG_HEADER, obj.__class__, obj._primarykey)
            if obj._primarykey == None:
                obj._primarykey =  uuid.uuid4()
            obj._frame = self
            self.newlyproduced[obj.__class__][obj._primarykey] = obj
            self.storebuffer[obj.__class__][obj._primarykey] = obj
        else:
            logger.error("%s Object not in dictionary: %s", LOG_HEADER, obj.__class__)

    def delete(self, t, oid):
        if oid in self.storebuffer[t]:
            o = self.storebuffer[t][oid]
            self.deletelist[t][o._primarykey] = o
            del self.storebuffer[t][oid]
        return True

    def findproperty(self, t, propname, value):
        for o in self.storebuffer[t].values():
            if getattr(o, propname) == value:
                return o
        return None

    def resolve_fk(self, obj, t):
        for propname, cls in t._foreignkeys.items():
            propvalue = getattr(obj, propname)
            fname = getattr(t, propname)._foreignprop._name
            if propvalue in self.fkdict[cls][fname]:
                primkey = self.fkdict[cls][fname][propvalue]
            else:
                #logger.error("Could not find property value in foreign key dictionary.")
                return None
            if primkey in self.storebuffer[cls]:
                newobj = self.storebuffer[cls][primkey]
                if hasattr(cls, '_foreignkeys'):
                    self.resolve_fk(newobj, cls)
                setattr(obj, propname, newobj)
            # Look up by name, just in case
            else:
                fname = getattr(obj.__class__, propname)._foreignprop._name
                for o in self.storebuffer[cls].values():
                    if propvalue == getattr(o, fname):
                        setattr(obj, propname, o)


    def get(self, t, primkey=None):
        self.track_changes = False
        try:
            if t not in self.storebuffer:
                self.__Logger.error("Could not find type %s in cache. Did you remember to add it as a gettter, setter, or producer in the simulator?")
                sys.exit(0)
            if primkey != None:
                obj = copy(self.storebuffer[t][primkey])
                if hasattr(t, '_foreignkeys'):
                    self.resolve_fk(obj, t)
                return obj
            else:
                res = []
                for obj in self.storebuffer[t].values():
                    res.append(self.get(t, obj._primarykey))
                return res
        except:
            self.__Logger.exception("Uncaught exception in Frame.Get")
            raise
        finally:
            self.track_changes = True


    def new(self, t):
        return copy(self.new_storebuffer[t].values())

    def deleted(self, t):
        return copy(self.del_storebuffer[t].values())

    def changed(self, t):
        if len(self.mod_storebuffer[t].keys()):
            #logger.debug("objects with IDs %s have changed", self.mod_storebuffer[t].keys())
            return copy(self.mod_storebuffer[t].values())
        else:
            return []

    def __deepcopy__(self, memo):
        return self
Exemplo n.º 13
0
class Frame(object):
    '''
    classdocs
    '''

    Store = None
    def __init__(self, store = None, process = False, settings = None):
        '''
        Constructor
        '''
        self.app = None
        self.timer = None
        self.interval = None
        self.track_changes = False
        self.step = 0
        self.curtime = time.time()
        self.__Logger = logger
        self.process = process
        self.settings = settings

        # Local storage for thread
        self.tlocal = None

        # Stores objects from store
        self.storebuffer = {}

        # Stores new objects since last pull
        self.new_storebuffer = {}
        # Stores modified objects since last pull
        self.mod_storebuffer = {}
        # Store removed objects since last pull
        self.del_storebuffer = {}

        # Types that are being observed (read) by this application
        self.observed = set()
        # Stores the types that can be updated (read/write) by app
        self.updated = set()
        # Stores the types that can be produced by app
        self.produced = set()
        # Stores the types that can be retrieved by app
        self.newlyproduced = {}
        # Store the types "tracked" by the app (i.e. creation and deletion only)
        self.tracked = {}

        # Properties that changed during this iteration
        self.changedproperties = {}

        # List of objects to be pushed to the store (no subsets)
        self.pushlist = {}

        # List of objects marked for deletion
        self.deletelist = {}

        # Holds a k,v storage of foreign keys (e.g. name -> id)
        self.fkdict = {}

        # Disables fetching of subsets
        self.subset_disable = set()

        # Keeps track of object ids received for subset queries that are not in buffer yet
        self.orphan_objids = {}

        self.timer = None
        self.step = 0
        self.thread = None
        self.encoder = CADISEncoder()
        if store:
            logger.debug("%s received store %s", LOG_HEADER, store)
            Frame.Store = store

    ######################################################
    ## Application -> Frame Functions
    ######################################################
    def add(self, obj):
        t = obj.__class__
        if t in self.storebuffer:
            # logger.debug("%s Creating new object %s.%s", LOG_HEADER, obj.__class__, obj._primarykey)
            if obj._primarykey == None:
                obj._primarykey = uuid.uuid4()
            obj._frame = self
            self.newlyproduced[t][obj._primarykey] = obj
            self.storebuffer[t][obj._primarykey] = obj
            # If we removed then readded in the same tick, make sure we don't send the remove anymore
            if t in self.deletelist and obj._primarykey in self.deletelist[t]:
                del self.deletelist[t][obj._primarykey]
        else:
            logger.error("%s Object not in dictionary: %s", LOG_HEADER, obj.__class__)

    def delete(self, t, oid):
        if oid in self.storebuffer[t]:
            o = self.storebuffer[t][oid]
            self.deletelist[t][o._primarykey] = o
            del self.storebuffer[t][oid]
            # If we added and removed in the same tick, make sure we don't send the add
            if t in self.newlyproduced and o._primarykey in self.newlyproduced[t]:
                del self.newlyproduced[t][o._primarykey]

        else:
            logger.warn("could not find object %s in storebuffer")
        return True

    def get(self, t, primkey=None):
        self.track_changes = False
        try:
            if t not in self.storebuffer:
                self.__Logger.error("Could not find type %s in cache. Did you remember to add it as a gettter, setter, or producer in the simulator?")
                sys.exit(0)
            if primkey != None:
                obj = self.storebuffer[t][primkey]
                if hasattr(t, '_foreignkeys'):
                    self._resolve_fk(obj, t)
                return obj
            else:
                res = []
                for obj in self.storebuffer[t].values():
                    res.append(self.get(t, obj._primarykey))
                return res
        except:
            self.__Logger.exception("Uncaught exception in Frame.Get")
            raise
        finally:
            self.track_changes = True


    def new(self, t):
        return self.new_storebuffer[t].values()

    def deleted(self, t):
        return self.del_storebuffer[t].values()

    def changed(self, t):
        return self.mod_storebuffer[t].values()

    def disable_subset(self, t):
        self.subset_disable.add(t)

    def enable_subset(self, t):
        if t in self.subset_disable:
            self.subset_disable.remove(t)

    def attach(self, app):
        self.app = app
        self.process_declarations(app)
        #self.app.initialize()

    def go(self, cmd_dict, timer=None):
        self.cmds = cmd_dict
        self.timer = timer
        self.runner = TimerThread(self, Frame.Store, cmd_dict, timer)
        if self.process:
            self.thread = Process(target=self.runner.run)
        else:
            self.thread = Thread(target=self.runner.run)
        self.thread.start()

    def join(self):
        self.thread.join()

    ######################################################
    ## Core Functions
    ######################################################
    def execute_Frame(self):
        try:
            self.pull()
            self.track_changes = True
            self.app.update()
            self.track_changes = False
            self.push()
            self.step += 1
            self.curtime = time.time()
        except:
            self.cmds["SimulatorPaused"] = True
            logger.exception("[%s] uncaught exception: ", self.app.__module__)

    def initialize_app(self):
        self.app.initialize()
        # Push initial objects the application has added.
        self.push()
        self._update_shared_status("Pushed")
        logger.warn("Application %s finished pushing", self.app._appname)
        it = 0
        done = False
        while(it < 300 and done == False and self.cmds["SimulatorShutdown"] == False):
            done = True
            for status in self._get_all_shared_statuses().values():
                if status != "Pushed":
                    done = False
                    break
            it += 1
            time.sleep(0.1)
        self.pull()

    def process_declarations(self, app):
        self.produced = app._producer
        self.updated = app._gettersetter
        self.tracked_only = set()

        self.observed = set()
        self.iterate_types = []
        if app._getter:
            self.observed = self.observed.union(app._getter)
        if app._gettersetter:
            self.observed = self.observed.union(app._gettersetter)
        if app._tracker:
            self.tracked_only = app._tracker.difference(self.observed)
            self.observed = self.observed.union(app._tracker)

        # Check for "parents" of subsets, they need to be fetched too
        for t in copy(self.observed):
            if t in schema_subsets:
                pt = setsof[t]
                self.observed.add(pt)

        for t in self.observed.union(self.produced).union(self.updated):
            # self.changedproperties[t] = {}
            self.storebuffer[t] = {}
            self.new_storebuffer[t] = {}
            self.mod_storebuffer[t] = {}
            self.del_storebuffer[t] = {}
            if hasattr(t, "_foreignkeys"):
                for propname, cls in t._foreignkeys.items():
                    fname = getattr(t, propname)._foreignprop._name
                    if not cls in self.fkdict:
                        self.fkdict[cls] = {}
                    if not fname in self.fkdict[cls]:
                        self.fkdict[cls][fname] = {}
            logger.debug("%s store buffer for type %s", LOG_HEADER, t)

        # When iterating for pull, we should request, in order:
        # (1) set items
        # (2) permutations
        # (3) subsets

        setitems = self.observed.intersection(schema_sets)
        permutations = self.observed.intersection(schema_permutationsets)
        subsets = self.observed.intersection(schema_subsets)
        for t in subsets:
            self.orphan_objids[t] = set()

        self.iterate_types.extend(setitems)
        self.iterate_types.extend(permutations)
        self.iterate_types.extend(subsets)

        # prepare structures for storing and pushing new, update, and deletion of objects
        for t in self.produced:
            self.newlyproduced[t] = {}
        for t in self.updated:
            self.deletelist[t] = {}
            self.pushlist[t] = {}

        if Frame.Store == None:
            logger.debug("%s creating new store", LOG_HEADER)
            if USE_REMOTE_STORE:
                Frame.Store = RemoteStore()
            else:
                Frame.Store = SimpleStore()
        setattr(self.app, "_appname", self.app.__class__.__name__)
        Frame.Store.register(self.app._appname)

    def stop(self):
        self.app.shutdown()

    ######################################################
    ## Frame -> Store Functions
    ######################################################
    @instrument
    def pull(self):
        tmpbuffer = {}
        for t in self.iterate_types:
            if t in self.subset_disable:
                continue

            self.new_storebuffer[t] = {}
            self.mod_storebuffer[t] = {}
            self.del_storebuffer[t] = {}

            # optimization: retrieve only objects that changed since last pull
            if hasattr(Frame.Store, "getupdated"):
                if t in self.tracked_only:
                    (new, mod, deleted) = Frame.Store.getupdated(t, self.app._appname, tracked_only=True)
                else:
                    (new, mod, deleted) = Frame.Store.getupdated(t, self.app._appname)
                if t in schema_subsets:
                    # Parent type of subset 't'
                    pt = setsof[t]

                    # Iterate for orphan keys from last pull
                    notfound = set()
                    for key in self.orphan_objids[t]:
                        if key in self.storebuffer[pt]:
                            o = copy(self.storebuffer[pt][key])
                            o.__class__ = t
                            self.storebuffer[t][key] = o
                            self.new_storebuffer[t][key] = o
                        else:
                            notfound.add(key)
                            self.__Logger.error("missing key %s for new object in subset list", key)
                    self.orphan_objids[t] = set().union(notfound)

                    for key in new:
                        # If the key is not in the store's buffer, its likely the object was created between
                        # the get for parent type and the current subset query. This is expected, just keep a reference
                        # to check for again on next pull.
                        if key in self.storebuffer[pt]:
                            o = copy(self.storebuffer[pt][key])
                            o.__class__ = t
                            self.storebuffer[t][key] = o
                            self.new_storebuffer[t][key] = o
                        else:
                            self.orphan_objids[t].add(key)
                    for key in mod:
                        o = copy(self.storebuffer[pt][key])
                        o.__class__ = t
                        self.storebuffer[t][key] = o
                        self.mod_storebuffer[t][key] = o
                    for key in deleted:
                        if key in self.storebuffer[pt]:
                            o = copy(self.storebuffer[pt][key])
                        elif key in self.del_storebuffer[pt]:
                            o = copy(self.del_storebuffer[pt][key])
                        else:
                            self.__Logger.warn("object %s was deleted, could find find a reference to give to application.", key)
                            o = CADIS()
                            o.ID = key
                        o.__class__ = t
                        del self.storebuffer[t][key]
                        self.del_storebuffer[t][key] = o
                else:
                    for o in new:
                        self.storebuffer[t][o._primarykey] = o
                        self.new_storebuffer[t][o._primarykey] = o
                        if o.__class__ in self.fkdict:
                            self._fkobj(o)
                    for o in mod:
                        self.storebuffer[t][o._primarykey] = o
                        self.mod_storebuffer[t][o._primarykey] = o
                        if o.__class__ in self.fkdict:
                            self._fkobj(o)
                    for key in deleted:
                        if key in self.storebuffer[t]:
                            o = self.storebuffer[t][key]
                            self.del_storebuffer[t][key] = o
                            del self.storebuffer[t][key]
                        if o.__class__ in self.fkdict:
                            self._delfkobj(o)

            else:
                tmpbuffer[t] = {}
                for o in Frame.Store.get(t):
                    tmpbuffer[t][o._primarykey] = o

                # TODO: Remove when Store does this
                # Added objects since last pull
                for o in tmpbuffer[t].values():
                    if o._primarykey not in self.storebuffer[t]:
                        # logger.debug("%s Found new object: %s", LOG_HEADER, o)
                        self.new_storebuffer[t][o._primarykey] = o
                    else:
                        # check if updated
                        orig = self.encoder.encode(self.storebuffer[t][o._primarykey])
                        new = self.encoder.encode(o)
                        if orig != new:
                            self.mod_storebuffer[t][o._primarykey] = o

                # Deleted objects since last pull
                for o in self.storebuffer[t].values():
                    if o._primarykey not in tmpbuffer[t]:
                        self.del_storebuffer[t][o._primarykey] = o

                self.storebuffer[t] = tmpbuffer[t]

    @instrument
    def push(self):
        cleartypes = set()
        for t in self.newlyproduced:
            if len(self.newlyproduced[t]) > 0:
                if hasattr(Frame.Store, "insert_all"):
                    Frame.Store.insert_all(t, self.newlyproduced[t].values(), self.app._appname)
                else:
                    for o in self.newlyproduced[t].values():
                        Frame.Store.insert(o, self.app._appname)
            if t not in self.observed:
                cleartypes.add(t)
            self.newlyproduced[t] = {}

        for t in self.deletelist:
            for o in self.deletelist[t].values():
                Frame.Store.delete(t, o._primarykey, self.app._appname)
                if t in self.pushlist and o._primarykey in self.pushlist[t]:
                    del self.pushlist[t][o._primarykey]
            self.deletelist[t] = {}

        Frame.Store.update_all(self.pushlist, self.app._appname)
        for t in cleartypes:
            self.storebuffer[t] = {}

    ######################################################
    ## Utility Functions
    ######################################################
    def buffersize(self):
        size = 0
        nobjects = 0
        for t in self.storebuffer:
            for o in self.storebuffer[t].values():
                nobjects += 1
                for prop in o._dimensions:
                    size += sys.getsizeof(prop)
        return (nobjects, size)

    def _fkobj(self, o):
        for propname in self.fkdict[o.__class__].keys():
            propvalue = getattr(o, propname)
            self.fkdict[o.__class__][propname][propvalue] = o.ID

    def _delfkobj(self, o):
        for propname in self.fkdict[o.__class__].keys():
            propvalue = getattr(o, propname)
            if propvalue in self.fkdict[o.__class__][propname]:
                del self.fkdict[o.__class__][propname][propvalue]

    def name2class(self, typeName):
        for t in self.storebuffer:
            if t.__name__ == typeName:
                return t

    def count(self, typeObj):
        if typeObj in self.storebuffer:
            return len(self.storebuffer[typeObj])
        else:
            return -1

    def _update_shared_status(self, newstatus):
        self.cmds["APP_" + self.app._appname] = newstatus

    def _get_all_shared_statuses(self):
        res = {}
        for k,v in self.cmds.items():
            if k.startswith("APP_"):
                res[k] = v
        return res

    def findproperty(self, t, propname, value):
        for o in self.storebuffer[t].values():
            if getattr(o, propname) == value:
                return o
        return None

    def set_property(self, t, o, v, n):
        # if not a gettersetter for this type, return
        if t not in self.updated:
            return

        # Newly produced items will be pushed entirely. Skip...
        if not o._primarykey:
            return

        if t in self.newlyproduced and o._primarykey in self.newlyproduced[t]:
            # logger.debug("[%s] Object ID %s in newly produced")
            return

        # Object not tracked by store yet. Ignore...
        if o._primarykey not in self.storebuffer[t]:
            # logger.debug("[%s] Object ID %s not being tracked yet")
            return

        if o._primarykey not in self.pushlist[t]:
            self.pushlist[t][o._primarykey] = {}

        # logger.debug("[%s] Object ID %s property %s being set to %s")
        # Save the property update
        self.pushlist[t][o._primarykey][n] = v
        # logger.debug("property %s of object %s (ID %s) set to %s", n, o, o._primarykey, v)

    def _resolve_fk(self, obj, t):
        for propname, cls in t._foreignkeys.items():
            propvalue = getattr(obj, propname)
            fname = getattr(t, propname)._foreignprop._name
            if propvalue in self.fkdict[cls][fname]:
                primkey = self.fkdict[cls][fname][propvalue]
            else:
                # logger.error("Could not find property value in foreign key dictionary.")
                return None
            if primkey in self.storebuffer[cls]:
                newobj = self.storebuffer[cls][primkey]
                if hasattr(cls, '_foreignkeys'):
                    self._resolve_fk(newobj, cls)
                setattr(obj, propname, newobj)
            # Look up by name, just in case
            else:
                fname = getattr(obj.__class__, propname)._foreignprop._name
                for o in self.storebuffer[cls].values():
                    if propvalue == getattr(o, fname):
                        setattr(obj, propname, o)



    def __deepcopy__(self, memo):
        return self
Exemplo n.º 14
0
class PythonRemoteStore(IStore):
    __Logger = logging.getLogger(__name__)

    def __init__(self, address="http://localhost:12000"):
        '''
        address: includes port. e.g. http://localhost:9000
        sim: identifier name for the simulator using this store
        '''
        self.encoder = CADISEncoder()
        if not address.endswith('/'):
            address += '/'
        self.address = address

    def insert(self, obj, sim):
        jsonobj = self.encoder.encode(obj)
        response = requests.put("%s%s/%s" %
                                (self.base_address, obj._FULLNAME, obj.ID),
                                data={'obj': jsonobj},
                                headers={
                                    'Content-Type': 'application/json',
                                    'InsertType': 'Single'
                                })
        return response

    def insert_all(self, t, list_obj, sim):
        objs = self.encoder.encode(list_obj)
        response = requests.put(self.base_address + t._FULLNAME,
                                data={'insert_list': objs},
                                headers={'InsertType': 'Multiple'},
                                stream=True)
        return response

    def get(self, typeObj):
        resp = requests.get(self.base_address + typeObj._FULLNAME)
        jsonlist = json.loads(resp.text)
        objlist = []
        for data in jsonlist:
            # obj = typeObj.__new__(typeObj)
            obj = CADIS()
            obj.__class__ = typeObj
            for dim in obj._dimensions:
                prop = getattr(obj, dim._name)
                if hasattr(prop, "__decode__"):
                    prop = prop.__decode__(data[dim._name])
                else:
                    prop = data[dim._name]
                setattr(obj, dim._name, prop)
            obj.ID = UUID(data["ID"])
            objlist.append(obj)
        return objlist

    def register(self, sim):
        self.base_address = self.address + sim + '/'
        resp = requests.put(self.base_address[:-1])
        self.sim = sim
        return resp

    def update_all(self, pushlist, sim):
        for t in pushlist:
            if len(pushlist[t]) > 0:
                tmp = pushlist[t]
                updates = self.encoder.encode(
                    {str(k): v
                     for k, v in tmp.items()})
                resp = requests.post(self.base_address + t._FULLNAME,
                                     data={'update_dict': updates})
            pushlist[t] = {}

    def create_obj(self, typeObj, data):
        try:
            obj = typeObj.__new__(typeObj)
            for dim in obj._dimensions:
                prop = getattr(obj, dim._name)
                if hasattr(prop, "__decode__"):
                    prop = prop.__decode__(data[dim._name])
                else:
                    prop = data[dim._name]
                setattr(obj, dim._name, prop)
            obj.ID = UUID(data["ID"])
        except Exception, e:
            self.__Logger.exception("Failed to create object from data %s",
                                    data)
        return obj
Exemplo n.º 15
0
 def __init__(self):
     '''
     Constructor
     '''
     self.address = "http://localhost:12000/"
     self.encoder = CADISEncoder()
Exemplo n.º 16
0
class Frame(object):
    '''
    classdocs
    '''

    Store = None

    def __init__(self, store=None):
        '''
        Constructor
        '''
        self.app = None
        self.timer = None
        self.interval = None
        self.track_changes = False
        self.step = 0
        self.curtime = time.time()
        self.__Logger = logger

        # Local storage for thread
        self.tlocal = None

        # Stores objects from store
        self.storebuffer = {}

        # Stores new objects since last pull
        self.new_storebuffer = {}
        # Stores modified objects since last pull
        self.mod_storebuffer = {}
        # Store removed objects since last pull
        self.del_storebuffer = {}

        # Types that are being observed (read) by this application
        self.observed = set()
        # Stores the types that can be updated (read/write) by app
        self.updated = set()
        # Stores the types that can be produced by app
        self.produced = set()
        # Stores the types that can be retrieved by app
        self.newlyproduced = {}

        # Properties that changed during this iteration
        self.changedproperties = {}

        # List of objects to be pushed to the store (no subsets)
        self.pushlist = {}

        # List of objects marked for deletion
        self.deletelist = {}

        # Holds a k,v storage of foreign keys (e.g. name -> id)
        self.fkdict = {}

        # Disables fetching of subsets
        self.subset_disable = set()

        self.timer = None
        self.step = 0
        self.thread = None
        self.encoder = CADISEncoder()
        if store:
            logger.debug("%s received store %s", LOG_HEADER, store)
            Frame.Store = store

    def attach(self, app):
        self.app = app
        self.process_declarations(app)
        self.app.initialize()

    def go(self):
        #self.timer = Timer(1.0, self.execute_Frame)
        #self.timer.start()
        self.thread = TimerThread(self)
        self.thread.start()
        #thread.join()
        #self.stop()
    def join(self):
        self.thread.join()

    def stop(self):
        if self.timer:
            self.timer.cancel()
        sys.exit(0)

    def disable_subset(self, t):
        self.subset_disable.add(t)

    def enable_subset(self, t):
        if t in self.subset_disable:
            self.subset_disable.remove(t)

    def process_declarations(self, app):
        self.produced = app._producer
        self.updated = app._gettersetter
        self.observed = set()
        if app._getter:
            self.observed = self.observed.union(app._getter)
        if app._gettersetter:
            self.observed = self.observed.union(app._gettersetter)

        for t in self.observed.union(self.produced).union(self.updated):
            #self.changedproperties[t] = {}
            self.storebuffer[t] = {}
            self.new_storebuffer[t] = {}
            self.mod_storebuffer[t] = {}
            self.del_storebuffer[t] = {}
            if hasattr(t, "_foreignkeys"):
                for propname, cls in t._foreignkeys.items():
                    fname = getattr(t, propname)._foreignprop._name
                    if not cls in self.fkdict:
                        self.fkdict[cls] = {}
                    if not fname in self.fkdict[cls]:
                        self.fkdict[cls][fname] = {}
            logger.debug("%s store buffer for type %s", LOG_HEADER, t)

        for t in self.produced:
            self.pushlist[t] = {}
            self.deletelist[t] = {}
            self.newlyproduced[t] = {}

        if Frame.Store == None:
            logger.debug("%s creating new store", LOG_HEADER)
            if USE_REMOTE_STORE:
                Frame.Store = RemoteStore()
            else:
                Frame.Store = SimpleStore()

        Frame.Store.register(self)

    def execute_Frame(self):
        try:
            self.pull()
            self.track_changes = True
            self.app.update()
            self.track_changes = False
            self.prepare_push()
            self.push()
            if self.timer:
                self.timer = Timer(1.0, self.execute_Frame)
                self.timer.start()
            self.step += 1
            self.curtime = time.time()
        except:
            logger.exception("uncaught exception: ")

    def pull(self):
        tmpbuffer = {}
        for t in self.observed.symmetric_difference(self.subset_disable):
            self.new_storebuffer[t] = {}
            self.mod_storebuffer[t] = {}
            self.del_storebuffer[t] = {}

            # if type is subset, it will always recalculate and return as new / updated
            # TODO: Allow caching of subsets to determine new / updated / deleted
            if hasattr(Frame.Store, "updated"):
                (new, mod, deleted) = Frame.Store.updated(t, self)
                for o in new:
                    self.storebuffer[t][o._primarykey] = o
                    self.new_storebuffer[t][o._primarykey] = o
                    if o.__class__ in self.fkdict:
                        for propname in self.fkdict[o.__class__].keys():
                            propvalue = getattr(o, propname)
                            self.fkdict[
                                o.__class__][propname][propvalue] = o.ID
                for o in mod:
                    self.storebuffer[t][o._primarykey] = o
                    self.mod_storebuffer[t][o._primarykey] = o
                    if o.__class__ in self.fkdict:
                        for propname in self.fkdict[o.__class__].keys():
                            propvalue = getattr(o, propname)
                            self.fkdict[
                                o.__class__][propname][propvalue] = o.ID
                for o in deleted:
                    if o._primarykey in self.storebuffer[t]:
                        del self.storebuffer[t][o._primarykey]
                        self.del_storebuffer[t][o._primarykey] = o
                    if o.__class__ in self.fkdict:
                        for propname in self.fkdict[o.__class__].keys():
                            propvalue = getattr(o, propname)
                            if propvalue in self.fkdict[o.__class__][propname]:
                                del self.fkdict[
                                    o.__class__][propname][propvalue]
            else:
                tmpbuffer[t] = {}
                for o in Frame.Store.get(t):
                    tmpbuffer[t][o._primarykey] = o

                #TODO: Remove when Store does this
                # Added objects since last pull
                for o in tmpbuffer[t].values():
                    if o._primarykey not in self.storebuffer[t]:
                        #logger.debug("%s Found new object: %s", LOG_HEADER, o)
                        self.new_storebuffer[t][o._primarykey] = o
                    else:
                        # check if updated
                        orig = self.encoder.encode(
                            self.storebuffer[t][o._primarykey])
                        new = self.encoder.encode(o)
                        if orig != new:
                            self.mod_storebuffer[t][o._primarykey] = o

                # Deleted objects since last pull
                for o in self.storebuffer[t].values():
                    if o._primarykey not in tmpbuffer[t]:
                        self.del_storebuffer[t][o._primarykey] = o

                self.storebuffer[t] = tmpbuffer[t]

    def prepare_push(self):
        #for t in self.changedproperties:
        #    push_key = None
        #    push_obj = None
        #    if t in subsets:
        #        push_key = setsof[t]
        #
        #    elif t in sets:
        #        push_key = t
        #        for o in self.changedproperties[t]:
        #            self.pushlist[t][o._primarykey] = o
        pass

    def push(self):
        for t in self.pushlist:
            for primkey in self.pushlist[t].keys():
                if primkey in self.newlyproduced[t]:
                    continue
                elif primkey in self.storebuffer[t]:
                    Frame.Store.update(t, primkey, self,
                                       self.pushlist[t][primkey])
                else:
                    logger.error(
                        "[%s] Missing object in store buffer. Object ID %s,  Pushlist: %s",
                        self.app, primkey, self.pushlist[t])
            self.pushlist[t] = {}

        for t in self.produced:
            if t in self.newlyproduced:
                for o in self.newlyproduced[t].values():
                    #logger.debug("[%s] Adding obj %s (ID %s) to store", self.app, o, o._primarykey)
                    Frame.Store.insert(o, self)
                self.newlyproduced[t] = {}

        for t in self.deletelist:
            for o in self.deletelist[t].values():
                Frame.Store.delete(t, o)

    def set_property(self, t, o, v, n):
        # Newly produced items will be pushed entirely. Skip...
        if not o._primarykey:
            return

        if t in self.newlyproduced and o._primarykey in self.newlyproduced[t]:
            #logger.debug("[%s] Object ID %s in newly produced")
            return

        # Object not tracked by store yet. Ignore...
        if o._primarykey not in self.storebuffer[t]:
            #logger.debug("[%s] Object ID %s not being tracked yet")
            return

        if o._primarykey not in self.pushlist[t]:
            self.pushlist[t][o._primarykey] = {}

        #logger.debug("[%s] Object ID %s property %s being set to %s")
        # Save the property update
        self.pushlist[t][o._primarykey][n] = v
        #logger.debug("property %s of object %s (ID %s) set to %s", n, o, o._primarykey, v)

    def add(self, obj):
        if obj.__class__ in self.storebuffer:
            #logger.debug("%s Creating new object %s.%s", LOG_HEADER, obj.__class__, obj._primarykey)
            if obj._primarykey == None:
                obj._primarykey = uuid.uuid4()
            obj._frame = self
            self.newlyproduced[obj.__class__][obj._primarykey] = obj
            self.storebuffer[obj.__class__][obj._primarykey] = obj
        else:
            logger.error("%s Object not in dictionary: %s", LOG_HEADER,
                         obj.__class__)

    def delete(self, t, oid):
        if oid in self.storebuffer[t]:
            o = self.storebuffer[t][oid]
            self.deletelist[t][o._primarykey] = o
            del self.storebuffer[t][oid]
        return True

    def findproperty(self, t, propname, value):
        for o in self.storebuffer[t].values():
            if getattr(o, propname) == value:
                return o
        return None

    def resolve_fk(self, obj, t):
        for propname, cls in t._foreignkeys.items():
            propvalue = getattr(obj, propname)
            fname = getattr(t, propname)._foreignprop._name
            if propvalue in self.fkdict[cls][fname]:
                primkey = self.fkdict[cls][fname][propvalue]
            else:
                #logger.error("Could not find property value in foreign key dictionary.")
                return None
            if primkey in self.storebuffer[cls]:
                newobj = self.storebuffer[cls][primkey]
                if hasattr(cls, '_foreignkeys'):
                    self.resolve_fk(newobj, cls)
                setattr(obj, propname, newobj)
            # Look up by name, just in case
            else:
                fname = getattr(obj.__class__, propname)._foreignprop._name
                for o in self.storebuffer[cls].values():
                    if propvalue == getattr(o, fname):
                        setattr(obj, propname, o)

    def get(self, t, primkey=None):
        self.track_changes = False
        try:
            if t not in self.storebuffer:
                self.__Logger.error(
                    "Could not find type %s in cache. Did you remember to add it as a gettter, setter, or producer in the simulator?"
                )
                sys.exit(0)
            if primkey != None:
                obj = copy(self.storebuffer[t][primkey])
                if hasattr(t, '_foreignkeys'):
                    self.resolve_fk(obj, t)
                return obj
            else:
                res = []
                for obj in self.storebuffer[t].values():
                    res.append(self.get(t, obj._primarykey))
                return res
        except:
            self.__Logger.exception("Uncaught exception in Frame.Get")
            raise
        finally:
            self.track_changes = True

    def new(self, t):
        return copy(self.new_storebuffer[t].values())

    def deleted(self, t):
        return copy(self.del_storebuffer[t].values())

    def changed(self, t):
        if len(self.mod_storebuffer[t].keys()):
            #logger.debug("objects with IDs %s have changed", self.mod_storebuffer[t].keys())
            return copy(self.mod_storebuffer[t].values())
        else:
            return []

    def __deepcopy__(self, memo):
        return self