def wrapper(*args, **kwds):
global db
global fs
if db is None or fs is None:
db,fs=misc.select_db(sys.argv)
args = (db,fs) + args
return f(*args, **kwds)
def wrapper(*args, **kwds):
global db
global fs
if db is None or fs is None:
db, fs = misc.select_db(sys.argv)
args = (db, fs) + args
return f(*args, **kwds)
def wrapper(*args, **kwds):
global db
global fs
global sysargs
if db is None or fs is None:
db, fs = misc.select_db(sysargs)
args = (db.new_context_copy(), fs.new_context_copy()) + args
return f(*args, **kwds)
def wrapper(*args, **kwds):
global db
global fs
global sysargs
if db is None or fs is None:
db,fs=misc.select_db(sysargs)
args = (db.new_context_copy(), fs.new_context_copy()) + args
return f(*args, **kwds)
def wrapper(*args, **kwds):
global db
global fs
global sysargs
if sysargs is None:
# Fake a sysargs object for a default
# this is for when we don't call this file directly
class A: pass
sysargs=A()
sysargs.db='mongo'
if db is None or fs is None:
db,fs=misc.select_db(sysargs)
args = (db,fs) + args
return f(*args, **kwds)
def get_commands():
from cgi import escape
import misc
db, fs = misc.select_db(None)
with open('commands.html', 'w') as f:
f.write("""
<html>
<STYLE TYPE="text/css">
pre { border: 1px solid}
</STYLE>
<body>""")
for m in db.database.input_messages.find():
f.write("<pre>")
f.write("<b>" + str(m.get('timestamp', None)) + "</b>")
f.write("\n")
code = m['content']['code']
s = escape(code if code is not None else 'None')
f.write(s.encode('utf-8'))
f.write("</pre>")
f.write("</body></html>")
def get_commands():
from cgi import escape
import misc
db, fs = misc.select_db(None)
with open('commands.html','w') as f:
f.write("""
<html>
<STYLE TYPE="text/css">
pre { border: 1px solid}
</STYLE>
<body>""")
for m in db.database.input_messages.find():
f.write("<pre>")
f.write("<b>"+str(m.get('timestamp',None))+"</b>")
f.write("\n")
code=m['content']['code']
s=escape(code if code is not None else 'None')
f.write(s.encode('utf-8'))
f.write("</pre>")
f.write("</body></html>")
default=singlecell_config.device_config.get('untrusted-cpu',-1),
help="CPU time (seconds) allotted to each session")
parser.add_option("--untrusted-mem", dest="untrusted_mem", type=float,
default=singlecell_config.device_config.get('untrusted-mem',-1),
help="Memory (MB) allotted to each session")
parser.add_option("-q", "--quiet", action="store_true", dest="quiet", help="Turn off most logging")
(sysargs,args)=parser.parse_args()
if sysargs.untrusted_account is "":
print "You must give an untrusted account we can ssh into using --untrusted-account"
sys.exit(1)
if sysargs.quiet:
util.LOGGING=False
db, fs = misc.select_db(sysargs)
keys=[b64encode(os.urandom(32)) if sysargs.print_cmd else os.urandom(32) for _ in (0,1)]
db_loop=MessageLoop(db, keys[0])
fs_loop=MessageLoop(fs, keys[1], isFS=True)
signal.signal(signal.SIGINT, signal_handler)
cwd=os.getcwd()
# TODO: make this a temporary file using python's (or the shell's) tempfile stuff
filename="/tmp/sage_shared_key%i"
options=dict(cwd=cwd, workers=sysargs.workers, db_port=db_loop.port, fs_port=fs_loop.port,
quiet='-q' if sysargs.quiet or util.LOGGING is False else '',
untrusted_python=sysargs.untrusted_python,
untrusted_cpu=sysargs.untrusted_cpu,
untrusted_mem=sysargs.untrusted_mem)
cmd="""cd %(cwd)s
%(untrusted_python)s device_process.py --db zmq --timeout 60 -w %(workers)s --cpu %(untrusted_cpu)f --mem %(untrusted_mem)f --dbaddress tcp://localhost:%(db_port)i --fsaddress=tcp://localhost:%(fs_port)i %(quiet)s\n"""%options
fs_file.close()
if len(file_list)>0:
file_list.sort()
#TODO: this message comes *after* the end of computation message from ipython
# either we need to slip it in before, or we need to have our own custom end-of-computation message.
db.add_messages(request_msg["_id"],[{'parent_header':header, 'sequence':sequence,
'msg_type':'extension',
'content':{'msg_type':'files',
'files':file_list}}])
shutil.rmtree(tempDir)
sequence+=1
db.add_messages(request_msg["_id"],[{'parent_header':header, 'sequence':sequence,
'msg_type':'extension',
'content':{'msg_type':'comp_end'}}])
#TODO: make polling interval a variable
time.sleep(0.1)
if __name__ == "__main__":
import misc
try:
from argparse import ArgumentParser
except ImportError:
from IPython.external import argparse
ArgumentParser=argparse.ArgumentParser
parser=ArgumentParser(description="Run one or more devices to process commands from the client.")
parser.add_argument("--db", choices=["mongo", "sqlalchemy"], help="Database to use")
parser.add_argument("-w", type=int, default=1, dest="workers", help="Number of workers to start.")
sysargs=parser.parse_args()
db, fs = misc.select_db(sysargs)
run_ip_pool(db, fs, workers=sysargs.workers)
out+=msg["content"]["data"]
changed=True
elif msg["msg_type"]=="pyout":
out+=msg["content"]["data"]["text/plain"]+"\n"
elif msg["msg_type"]=="status" and msg["content"]["execution_state"]=="idle":
done=True
elif msg["msg_type"]=="pyerr":
err+=new_stream("error", printout=False, **msg["content"])
if changed or done:
db.set_output(X["_id"], make_output_json(out+err, done))
if done:
break
time.sleep(0.1)
if __name__ == "__main__":
import misc
if "ipython" in sys.argv:
db, fs = misc.select_db([""])
run_ip_device()
else:
# argv[1] is number of workers
# argv[2] is "mongo" (default) or "sqlite"
db, fs = misc.select_db([""] if len(sys.argv)<3 else ["", sys.argv[2]])
if len(sys.argv)<=1:
workers=1
else:
workers=int(sys.argv[1])
outQueue=Queue()
fslock=Lock()
run(db, fs, workers=workers)
"execution_state"] == "idle":
done = True
elif msg["msg_type"] == "pyerr":
err += new_stream("error",
printout=False,
**msg["content"])
if changed or done:
db.set_output(X["_id"], make_output_json(out + err, done))
if done:
break
time.sleep(0.1)
if __name__ == "__main__":
import misc
if "ipython" in sys.argv:
db, fs = misc.select_db([""])
run_ip_device()
else:
# argv[1] is number of workers
# argv[2] is "mongo" (default) or "sqlite"
db, fs = misc.select_db(
[""] if len(sys.argv) < 3 else ["", sys.argv[2]])
if len(sys.argv) <= 1:
workers = 1
else:
workers = int(sys.argv[1])
outQueue = Queue()
fslock = Lock()
run(db, fs, workers=workers)
