def start_site(name,port,public=False,sudo=False):
"""
"""
#---start django
site_dn = os.path.join('site',name)
if not os.path.isdir(site_dn):
raise Exception('missing project named "%s". did you forget to connect it?'%name)
#---if public we require an override port so that users are careful
if public:
public_details = get_public_ports(name)
port = public_details['port_site']
#---previously got user/group from the public dictionary but now we just use the current user
user,group = username,groupname
check_port(port)
#---for some reason you have to KILL not TERM the runserver
#---! replace runserver with something more appropriate? a real server?
lock = 'pid.%s.site.lock'%name
log = log_site%name
if not public:
cmd = 'python %s runserver 0.0.0.0:%s'%(os.path.join(os.getcwd(),site_dn,'manage.py'),port)
else:
#---! hard-coded development static paths
cmd = ('%senv/envs/py2/bin/mod_wsgi-express start-server '%('sudo ' if sudo else '')+
'--port %d site/%s/%s/wsgi.py --user %s --group %s '+
'--python-path site/%s %s')%(port,name,name,user,group,name,
('--url-alias /static %s'%('interface/static' if not public else 'site/%s/static_root'%name)))
auth_fn = os.path.join('site',name,name,'wsgi_auth.py')
if os.path.isfile(auth_fn): cmd += ' --auth-user-script=%s'%auth_fn
backrun(cmd=cmd,log=log,stopper=lock,killsig='KILL',sudo=sudo,
scripted=False,kill_switch_coda='rm %s'%lock,notes=('# factory run is public' if public else None))
if public: chown_user(log)
return lock,log
def start_notebook(name,port,public=False,sudo=False):
"""
"""
#---if public we require an override port so that users are careful
if public:
public_details = get_public_ports(name)
port,notebook_ip = [public_details[i] for i in ['port_notebook','notebook_ip']]
#---! low ports are on. to turn them off remove False below and
if port<=1024: raise Exception('cannot use port %d for this project. '%port+
'even public projects need high notebook ports for security reasons. '+
'you will need to run `make connect <name> public` after you fix the ports')
if not os.path.isdir(os.path.join('site',name)):
raise Exception('cannot find site for %s'%name)
#---note that TERM safely closes the notbook server
lock = 'pid.%s.notebook.lock'%name
log = log_notebook%name
# root location to serve the notebook
note_root = os.path.join(os.getcwd(),'calc',name)
#! demo to try to serve higher for automacs simulations via notebook
note_root = os.getcwd()
#---higher rates (increased from 10M to 10**10 for XTC downloads)
rate_cmd = '--NotebookApp.iopub_data_rate_limit=10000000000'
#---if you want django data in IPython, use:
#---...'python site/%s/manage.py shell_plus --notebook --no-browser'%name,
#---! we never figured out how to set ports, other jupyter settings, with shell_plus
#---! jupyter doesn't recommend allowing root but we do so here so you can call
#---! `sudo make run <name> public` which prevents us from having to add sudo ourselves
if not public:
cmd = 'jupyter notebook --no-browser --port %d --port-retries=0 %s--notebook-dir="%s"'%(
port,('%s '%rate_cmd if rate_cmd else ''),note_root)
#---note that without zeroing port-retries, jupyter just tries random ports nearby (which is bad)
else:
#! note try: jupyter notebook password --generate-config connections/config_jupyter_actinlink.py
#! ... which will give you a one-per-machine password to link right to the notebook
#! ... note that you must write that password to logs/token.PROJECT_NAME with a trailing space!
#---! unsetting this variable because some crazy run/user error
if 'XDG_RUNTIME_DIR' in os.environ: del os.environ['XDG_RUNTIME_DIR']
cmd = (('sudo -i -u %s '%username if sudo else '')+'%s '%(
os.path.join(os.getcwd(),'env/envs/py2/bin/jupyter-notebook'))+
('--user=%s '%username if sudo else '')+(' %s '%rate_cmd if rate_cmd else '')+
'--port-retries=0 '+'--port=%d --no-browser --ip="%s" --notebook-dir="%s"'%(port,
notebook_ip if not public_details.get('jupyter_localhost',False) else 'localhost',
note_root))
backrun(cmd=cmd,log=log,stopper=lock,killsig='TERM',
scripted=False,kill_switch_coda='rm %s'%lock,sudo=sudo,
notes=('# factory run is public' if public else None))
if sudo: chown_user(log)
#! see the note above about jupyter notebook password. this is a hack to get the password working
token_log = log_token%name
if public and os.path.isfile(token_log):
time.sleep(3) # sleep to be sure that the log is ready
with open(token_log) as fp: token = fp.read().strip()
# route the token to the notebook log where it is picked up in the usual way
with open(log,'a') as fp: fp.write('\n[TOKEN] http://localhost:8888/?token=%s \n'%token)
#---note that the calling function should make sure the notebook started
return lock,log
def start_cluster(name,public=False,sudo=False):
"""
"""
#---start the cluster. argument is the location of the kill switch for clean shutdown
#---! eventually the cluster should move somewhere safe and the kill switches should be hidden
lock = 'pid.%s.cluster.lock'%name
log = log_cluster%name
#---! ...make shutdown should manage the clean shutdown
#---cluster never requires sudo but we require sudo to run publicly so we pass it along
#---! run the cluster as the user when running public?
backrun(cmd='python -u mill/cluster_start.py %s'%lock,
log=log,stopper=lock,killsig='INT',scripted=False,sudo=sudo,
notes=('# factory run is public' if public else None))
if sudo: chown_user(log)
return lock,log
def testcluster():
"""
"""
confirm_env()
backrun(cmd='python -u mill/start.py stop-cluster.sh',
log='cluster.log',
stopper='clusterhalt.sh',
killsig='INT')
#---! wait for the cluster to get ready
time.sleep(2)
if False:
#---submit a few jobs
for j in range(10):
#---write a job request
stamp = datetime.datetime.now().strftime('%Y%m%d%H%M%S.%f')
with open('cluster/%s.req' % stamp, 'w') as fp:
script = '\n'.join(
['echo "WAITING"', 'sleep 12', 'echo "DERPPP"'])
fp.write(json.dumps({'bash': script}))
time.sleep(0.05)
def run(self,cmd,log,kill_switch_coda_extras=None,use_bash=False):
"""
Generic computation which uses the logging functionality.
Used for `make compute` and the thumbnailer.
"""
self.avail()
self.log_fn = log
kwargs = dict(log=self.log_fn,stopper=self.lock_fn_abs,
cwd=self.cwd,killsig='KILL',scripted=False,kill_switch_coda='rm %s%s'%(self.lock_fn_abs,
'\n%s'%kill_switch_coda_extras if kill_switch_coda_extras else ''))
#---single command
if not use_bash:
cmd = '%s kill_switch="%s"'%(cmd,self.lock_fn_abs)
backrun(cmd=cmd,**kwargs)
#---bash script
else:
print('USING BASH')
print(cmd)
backrun(bash=cmd,**kwargs)
self.log_fn_abs = os.path.join(self.cwd,self.log_fn)
self.state = 'running'