def test_dbi(self):
instance = str(uuid.uuid4())
db = remus.db.connect(config_test.DEFAULT_DB)
db.createInstance(instance, {"info" : "test"} )
table_a_ref = remus.db.TableRef(instance, "/test1")
db.createTable(table_a_ref, { "info" : "other"} )
db.addData(table_a_ref, "key_1", {"data" : "key_1"})
db.addData(table_a_ref, "key_2", {"data" : "key_2"})
db.flush()
i = {}
for key in db.listKeys(table_a_ref):
i[key] = True
assert "key_1" in i
assert "key_2" in i
count = 0
for key, val in db.listKeyValue(table_a_ref):
assert val['data'] == key
count += 1
assert count == 2
print db.hasTable(table_a_ref)
assert db.hasTable(table_a_ref) == True
db.deleteTable(table_a_ref)
assert db.hasTable(table_a_ref)==False
table_b_ref = remus.db.TableRef(instance, "/test_b")
db.createTable(table_b_ref, {})
db.addData(table_b_ref, "key_1", {})
db.copyTo("run_tests.py", table_b_ref, "key_1", "run_tests.py")
db.flush()
db.copyFrom("tmp.out", table_b_ref, "key_1", "run_tests.py")
assert filecmp.cmp("tmp.out", "run_tests.py")
assert db.hasAttachment(table_b_ref, "key_1", "run_tests.py")
assert db.hasAttachment(table_b_ref, "key_1", "blabla.py") == False
assert db.listAttachments(table_b_ref, "key_1") == ["run_tests.py"]
db.deleteInstance(instance)
def test_dbi(self):
instance = str(uuid.uuid4())
db = remus.db.connect(config_test.DEFAULT_DB)
db.createInstance(instance, {"info": "test"})
table_a_ref = remus.db.TableRef(instance, "/test1")
db.createTable(table_a_ref, {"info": "other"})
db.addData(table_a_ref, "key_1", {"data": "key_1"})
db.addData(table_a_ref, "key_2", {"data": "key_2"})
db.flush()
i = {}
for key in db.listKeys(table_a_ref):
i[key] = True
assert "key_1" in i
assert "key_2" in i
count = 0
for key, val in db.listKeyValue(table_a_ref):
assert val['data'] == key
count += 1
assert count == 2
print db.hasTable(table_a_ref)
assert db.hasTable(table_a_ref) == True
db.deleteTable(table_a_ref)
assert db.hasTable(table_a_ref) == False
table_b_ref = remus.db.TableRef(instance, "/test_b")
db.createTable(table_b_ref, {})
db.addData(table_b_ref, "key_1", {})
db.copyTo("run_tests.py", table_b_ref, "key_1", "run_tests.py")
db.flush()
db.copyFrom("tmp.out", table_b_ref, "key_1", "run_tests.py")
assert filecmp.cmp("tmp.out", "run_tests.py")
assert db.hasAttachment(table_b_ref, "key_1", "run_tests.py")
assert db.hasAttachment(table_b_ref, "key_1", "blabla.py") == False
assert db.listAttachments(table_b_ref, "key_1") == ["run_tests.py"]
db.deleteInstance(instance)
def test_file(self):
instance = str(uuid.uuid4())
db = remus.db.connect(config_test.DEFAULT_DB)
db.createInstance(instance, {"info" : "test"} )
table_a_ref = remus.db.TableRef(instance, "/test_file")
db.createFile(table_a_ref, { "info" : "other"} )
db.copyTo("run_tests.py", table_a_ref)
assert db.hasTable(table_a_ref) == False
assert db.hasFile(table_a_ref) == True
db.copyFrom("tmp.out", table_a_ref)
assert filecmp.cmp("tmp.out", "run_tests.py")
db.deleteFile(table_a_ref)
assert db.hasFile(table_a_ref)==False
db.deleteInstance(instance)
def run(self):
"""
The work running method
"""
db = remus.db.connect( self.config.dbpath )
if not os.path.exists(self.config.workdir):
os.mkdir(self.config.workdir)
tmpdir = tempfile.mkdtemp(dir=self.config.workdir)
app = self.appletPath.split(":")
instRef = remus.db.TableRef(app[0], app[1])
parentName = re.sub("@request$", "", app[1] )
appName = app[2]
appPath = parentName + app[2]
logging.info("instanceREF: " + str(instRef))
logging.info("parent: " + parentName)
logging.info("appname: " + appName)
runVal = None
for val in db.getValue(instRef, appName):
runVal = val
if runVal is None:
raise Exception("Instance Entry not found")
#look at the submission record to determine with environments need
#to be copied to the workdir
if "_environment" in runVal:
envRef = remus.db.TableRef(instRef.instance, "@environment")
for env in runVal["_environment"]:
for name in db.listAttachments(envRef, env):
opath = os.path.join(tmpdir, name)
logging.info("client copy: " + opath)
if not os.path.exists(os.path.dirname(opath)):
os.makedirs(os.path.dirname(opath))
db.copyFrom(opath, envRef, env, name)
#add work dir to path list, so code can be used for pickel and object instances
sys.path.insert(0, tmpdir)
manager = remus.manage.Manager(self.config)
errorRef = remus.db.TableRef(instRef.instance, parentName + "@error")
if '_submitInit' in runVal:
try:
runClass = runVal['_submitInit']
sys.path.insert( 0, os.path.abspath(tmpdir) )
tmp = runClass.split('.')
mod = __import__(tmp[0])
cls = mod
for n in tmp[1:]:
cls = getattr(cls, n)
if issubclass(cls, remus.SubmitTarget):
obj = cls()
else:
obj = cls(runVal)
except Exception:
db.addData(errorRef, appName, {'error' : str(traceback.format_exc())})
return
elif db.hasAttachment(instRef, appName, "pickle"):
db.copyFrom(tmpdir + "/pickle", instRef, appName, "pickle")
handle = open(tmpdir + "/pickle")
try:
obj = pickle.loads(handle.read())
except Exception:
db.addData(errorRef, appName, {'error' : str(traceback.format_exc()), 'host' : socket.gethostname(), 'dir' : tmpdir})
db.flush()
return
handle.close()
cwd = os.getcwd()
os.chdir(tmpdir)
obj.__setpath__(instRef.instance, appPath)
obj.__setmanager__(manager)
self.inputList = []
self.outputList = []
manager._set_callback(self)
if '_outTable' in runVal:
obj.__outTableRef__ = remus.db.TableRef(instRef.instance, runVal['_outTable'])
obj.__outTable__ = remus.db.table.WriteTable(db, obj.__outTableRef__)
logging.debug("Starting user code")
stdout_path = os.path.abspath(os.path.join(tmpdir, "remus.stdout"))
stderr_path = os.path.abspath(os.path.join(tmpdir, "remus.stderr"))
child = os.fork()
if not child:
stdout_backup = os.dup(1)
stderr_backup = os.dup(2)
os.close(1)
os.open(stdout_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
os.close(2)
os.open(stderr_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
try:
if isinstance(obj, remus.SubmitTarget):
obj.run(runVal)
elif isinstance(obj, remus.MultiApplet):
obj.__run__()
else:
obj.run()
os.close(1)
os.dup(stdout_backup)
os.close(2)
os.dup(stderr_backup)
except Exception:
os.close(1)
os.dup(stdout_backup)
os.close(2)
os.dup(stderr_backup)
logging.error("user code error")
db.addData(errorRef, appName, {'error' : str(traceback.format_exc()), 'time' : datetime.datetime.now().isoformat(), 'host' : socket.gethostname()})
db.copyTo(stdout_path, errorRef, appName, "stdout")
db.copyTo(stderr_path, errorRef, appName, "stderr")
obj.__close__()
doneRef = remus.db.TableRef(instRef.instance, parentName + "@done")
db.addData(doneRef, appName, { 'time' : datetime.datetime.now().isoformat(), 'input' : self.inputList, 'output' : self.outputList })
db.flush()
sys.exit(0)
else:
pid, stat = os.waitpid(child,0)
#send our writes to the DB
db.flush()
logging.info("user code done")
#clean up workspace
os.chdir(cwd)
shutil.rmtree(tmpdir)
#if we have spawned local children, take care of them using the process manager
if self.local_children:
logging.info("Starting Local Process manager")
cConf = Config(self.config.dbpath, "process", workdir=self.config.workdir)
cManager = Manager(cConf)
cManager.wait(instRef.instance, appPath + "/@request")