Пример #1
0
    def test_dbi(self):
        
        instance = str(uuid.uuid4())
        db = remus.db.connect(config_test.DEFAULT_DB)
        
        db.createInstance(instance, {"info" : "test"} )
        
        table_a_ref = remus.db.TableRef(instance, "/test1")
        db.createTable(table_a_ref, { "info" : "other"}    )
        
        db.addData(table_a_ref, "key_1", {"data" : "key_1"})
        db.addData(table_a_ref, "key_2", {"data" : "key_2"})
        
        db.flush()
        
        i = {}
        for key in db.listKeys(table_a_ref):
            i[key] = True
        
        assert "key_1" in i
        assert "key_2" in i

        count = 0
        for key, val in db.listKeyValue(table_a_ref):
            assert val['data'] == key
            count += 1
        assert count == 2
        
        print db.hasTable(table_a_ref)
        assert db.hasTable(table_a_ref) == True
        
        db.deleteTable(table_a_ref)
        
        assert db.hasTable(table_a_ref)==False
                
        table_b_ref = remus.db.TableRef(instance, "/test_b")
        db.createTable(table_b_ref, {})
        db.addData(table_b_ref, "key_1", {})
        db.copyTo("run_tests.py", table_b_ref, "key_1", "run_tests.py")
        
        db.flush()
        
        db.copyFrom("tmp.out", table_b_ref, "key_1", "run_tests.py")
        
        assert filecmp.cmp("tmp.out", "run_tests.py")
        
        assert db.hasAttachment(table_b_ref, "key_1", "run_tests.py")
        assert db.hasAttachment(table_b_ref, "key_1", "blabla.py") == False
        
        assert db.listAttachments(table_b_ref, "key_1") == ["run_tests.py"]
        
        
        db.deleteInstance(instance)
Пример #2
0
    def test_dbi(self):

        instance = str(uuid.uuid4())
        db = remus.db.connect(config_test.DEFAULT_DB)

        db.createInstance(instance, {"info": "test"})

        table_a_ref = remus.db.TableRef(instance, "/test1")
        db.createTable(table_a_ref, {"info": "other"})

        db.addData(table_a_ref, "key_1", {"data": "key_1"})
        db.addData(table_a_ref, "key_2", {"data": "key_2"})

        db.flush()

        i = {}
        for key in db.listKeys(table_a_ref):
            i[key] = True

        assert "key_1" in i
        assert "key_2" in i

        count = 0
        for key, val in db.listKeyValue(table_a_ref):
            assert val['data'] == key
            count += 1
        assert count == 2

        print db.hasTable(table_a_ref)
        assert db.hasTable(table_a_ref) == True

        db.deleteTable(table_a_ref)

        assert db.hasTable(table_a_ref) == False

        table_b_ref = remus.db.TableRef(instance, "/test_b")
        db.createTable(table_b_ref, {})
        db.addData(table_b_ref, "key_1", {})
        db.copyTo("run_tests.py", table_b_ref, "key_1", "run_tests.py")

        db.flush()

        db.copyFrom("tmp.out", table_b_ref, "key_1", "run_tests.py")

        assert filecmp.cmp("tmp.out", "run_tests.py")

        assert db.hasAttachment(table_b_ref, "key_1", "run_tests.py")
        assert db.hasAttachment(table_b_ref, "key_1", "blabla.py") == False

        assert db.listAttachments(table_b_ref, "key_1") == ["run_tests.py"]

        db.deleteInstance(instance)
Пример #3
0
    def run(self):
        """
        The work running method
        
        """
        db = remus.db.connect( self.config.dbpath )
        if not os.path.exists(self.config.workdir):
            os.mkdir(self.config.workdir)
        tmpdir = tempfile.mkdtemp(dir=self.config.workdir)
        
        app = self.appletPath.split(":")
        
        instRef = remus.db.TableRef(app[0], app[1])
        parentName = re.sub("@request$", "", app[1] )
        appName = app[2]
        appPath = parentName + app[2]
        
        logging.info("instanceREF: " + str(instRef))
        logging.info("parent: " +  parentName)
        logging.info("appname: " + appName)
        
        runVal = None
        for val in db.getValue(instRef, appName):
            runVal = val
        if runVal is None:
            raise Exception("Instance Entry not found")
        
        #look at the submission record to determine with environments need 
        #to be copied to the workdir
        if "_environment" in runVal:
            envRef = remus.db.TableRef(instRef.instance, "@environment")
            for env in runVal["_environment"]:
                for name in db.listAttachments(envRef, env):
                    opath = os.path.join(tmpdir, name)
                    logging.info("client copy: " + opath)
                    if not os.path.exists(os.path.dirname(opath)):
                        os.makedirs(os.path.dirname(opath))
                    db.copyFrom(opath, envRef, env, name) 
            #add work dir to path list, so code can be used for pickel and object instances
            sys.path.insert(0, tmpdir)
            
            
        manager = remus.manage.Manager(self.config)
        errorRef = remus.db.TableRef(instRef.instance, parentName + "@error")

        if '_submitInit' in runVal:
            try:
                runClass = runVal['_submitInit']
                sys.path.insert( 0, os.path.abspath(tmpdir) )            
                tmp = runClass.split('.')
                mod = __import__(tmp[0])
                cls = mod
                for n in tmp[1:]:
                    cls = getattr(cls, n)
                if issubclass(cls, remus.SubmitTarget):
                    obj = cls()
                else:
                    obj = cls(runVal)
            except Exception:
                db.addData(errorRef, appName, {'error' : str(traceback.format_exc())})
                return  
        elif db.hasAttachment(instRef, appName, "pickle"):
            db.copyFrom(tmpdir + "/pickle", instRef, appName, "pickle")
            handle = open(tmpdir + "/pickle")
            try:
                obj = pickle.loads(handle.read())
            except Exception:
                db.addData(errorRef, appName, {'error' : str(traceback.format_exc()), 'host' : socket.gethostname(), 'dir' : tmpdir})
                db.flush()
                return
            handle.close()
        
        cwd = os.getcwd()
        os.chdir(tmpdir)
        obj.__setpath__(instRef.instance, appPath)
        obj.__setmanager__(manager)
        self.inputList = []
        self.outputList = []
        manager._set_callback(self)
        if '_outTable' in runVal:
            obj.__outTableRef__ = remus.db.TableRef(instRef.instance, runVal['_outTable'])
            obj.__outTable__ =  remus.db.table.WriteTable(db, obj.__outTableRef__)
        logging.debug("Starting user code")
        
        stdout_path = os.path.abspath(os.path.join(tmpdir, "remus.stdout"))
        stderr_path = os.path.abspath(os.path.join(tmpdir, "remus.stderr"))
        
        child = os.fork()
        if not child:
            stdout_backup = os.dup(1)
            stderr_backup = os.dup(2)
            os.close(1)
            os.open(stdout_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
            os.close(2)
            os.open(stderr_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
            try:
                if isinstance(obj, remus.SubmitTarget):
                    obj.run(runVal)
                elif isinstance(obj, remus.MultiApplet):
                    obj.__run__()
                else:
                    obj.run()
                os.close(1)
                os.dup(stdout_backup)
                os.close(2)
                os.dup(stderr_backup)                
            except Exception:
                os.close(1)
                os.dup(stdout_backup)
                os.close(2)
                os.dup(stderr_backup)                
                logging.error("user code error")
                db.addData(errorRef, appName, {'error' : str(traceback.format_exc()), 'time' : datetime.datetime.now().isoformat(), 'host' : socket.gethostname()})
                db.copyTo(stdout_path, errorRef, appName, "stdout")
                db.copyTo(stderr_path, errorRef, appName, "stderr")
            obj.__close__()
            doneRef = remus.db.TableRef(instRef.instance, parentName + "@done")
            db.addData(doneRef, appName, { 'time' : datetime.datetime.now().isoformat(), 'input' : self.inputList, 'output' : self.outputList })
            db.flush()
            sys.exit(0)
        else:
            pid, stat = os.waitpid(child,0)
            
        
        #send our writes to the DB
        db.flush()
        
        logging.info("user code done")
        
        #clean up workspace
        os.chdir(cwd)
        shutil.rmtree(tmpdir)
        
        
        #if we have spawned local children, take care of them using the process manager
        if self.local_children:
            logging.info("Starting Local Process manager")
            cConf = Config(self.config.dbpath, "process", workdir=self.config.workdir)
            cManager = Manager(cConf)
            cManager.wait(instRef.instance, appPath + "/@request")