def main():
sift_root = init.env_var_or_exit('SIFT_ROOT')
ipc_root = init.env_var_or_exit('IPC_ROOT')
dag = load_dag(sift_root)
threads = {}
sockets = []
node_indexes = sys.argv[1:]
if len(node_indexes) == 0:
print('no nodes to execute')
return 1
dry = os.environ.get('DRY', 'false')
if dry == 'true':
return 0
for i in map(int, node_indexes):
src = os.path.join(sift_root, dag['dag']['nodes'][i]['implementation']['python'])
print('loading ' + src)
m = new_module(i, src)
# Create nanomsg socket.
addr = 'ipc://%s/%d.sock'% (ipc_root, i)
s = Socket(REP)
s.recv_max_size = -1
s.connect(addr)
print('connected to '+ addr)
sockets.append(s)
# Launch request handler.
t = threading.Thread(target=listen_and_reply, args=(s, m.compute))
t.daemon = True
t.start()
threads[i] = t
try:
while True:
time.sleep(1)
for i, thr in threads.items():
if not thr.isAlive():
raise Exception('thread of node with index %d is dead' % i)
finally:
print('closing sockets')
for s in sockets: s.close()
def load_dag(sift_root):
sift_json = init.env_var_or_exit('SIFT_JSON')
return json.load(open(os.path.join(sift_root, sift_json)))
when a requirements file is found, we are installing the listed
libraries into a site-packages directory next to the source file.
This site-packages dir is perpended to the Python path by run.py
when loading the implementation.
"""
from __future__ import print_function
import json
import os
import os.path
import pip
import sys
import init
sr = init.env_var_or_exit('SIFT_ROOT')
sj = init.env_var_or_exit('SIFT_JSON')
ir = init.env_var_or_exit('IPC_ROOT')
sift = json.load(open(os.path.join(sr, sj)))
for n in sift['dag']['nodes']:
if 'implementation' in n and 'python' in n['implementation']:
d = os.path.dirname(n['implementation']['python'])
requirements_file = os.path.join(sr, d, 'requirements.txt')
if os.path.exists(requirements_file):
td = os.path.join(sr, d, 'site-packages')
ret = pip.main(['install', '--target='+td, '-r', requirements_file])
if ret != 0:
print('pip install returned code: %s' % ret)
sys.exit(ret)
