def setup_worker_userdata(configs, headnode_ip):
if configs['SERVICE_TYPE'] == 'slurm':
userdata = configurator.getstring("worker_slurm")
elif configs['SERVICE_TYPE'] == 'coasters':
userdata = configurator.getstring("worker_coasters")
else:
logging.error("Unknown SERVICE_TYPE:{0}\nCannot proceed, failing".format(configs['SERVICE_TYPE']))
exit(-1);
userdata = userdata.replace("SET_HEADNODE_IP", headnode_ip)
concurrency= ""
if 'WORKER_CONCURRENCY' in configs:
concurrency = "-c " + str(configs['WORKER_CONCURRENCY'])
userdata = userdata.replace("SET_CONCURRENCY", concurrency)
if 'WORKER_INIT_SCRIPT' in configs :
if not os.path.isfile(configs['WORKER_INIT_SCRIPT']):
print "Unable to read WORKER_INIT_SCRIPT"
exit(-1);
init_string = open(configs['WORKER_INIT_SCRIPT'], 'r').read()
userdata = userdata.replace("#WORKER_INIT_SCRIPT", init_string)
logging.info("Worker userdata : %s", userdata)
userdata = userdata.replace("#ETC_LINE", "{0}\t{1}".format(headnode_ip, HEADNODE_NAME))
logging.debug("USERDATA:\n{0}\n".format(userdata))
return userdata
def setup_headnode_userdata(configs):
if configs['SERVICE_TYPE'] == 'slurm':
userdata = configurator.getstring("headnode_slurm")
elif configs['SERVICE_TYPE'] == 'coasters':
userdata = configurator.getstring("headnode_coasters")
else:
logging.error("Unknown SERVICE_TYPE:{0}\nCannot proceed, failing".format(configs['SERVICE_TYPE']))
exit(-1);
return userdata
def setup_headnode_userdata(configs):
if configs['SERVICE_TYPE'] == 'slurm':
userdata = configurator.getstring("headnode_slurm")
elif configs['SERVICE_TYPE'] == 'coasters':
userdata = configurator.getstring("headnode_coasters")
elif configs['SERVICE_TYPE'] == 'turbine':
userdata = configurator.getstring("headnode_turbine")
userdata = userdata.replace("PRIVATE_KEY", open("./turbine_rsa").read())
else:
logging.error("Unknown SERVICE_TYPE:{0}\nCannot proceed, failing".format(configs['SERVICE_TYPE']))
exit(-1);
return userdata
def start_headnode(driver, configs):
userdata = configurator.getstring("headnode")
# Check if headnode
nodes = driver.list_nodes()
headnode = False
for node in nodes:
if node.name == "headnode" and node.state == NodeState.RUNNING:
headnode = node
print "INFO: Headnode is RUNNING"
return 0
size = NodeSize(id=configs['HEADNODE_MACHINE_TYPE'], name='headnode',
ram=None, disk=None, bandwidth=None, price=None, driver=driver)
image = NodeImage(id=configs['HEADNODE_IMAGE'], name=None, driver=driver)
node = driver.create_node(name='headnode',
image=image,
size=size,
ex_keyname=configs['AWS_KEYPAIR_NAME'],
ex_securitygroup=configs['SECURITY_GROUP'],
ex_userdata=userdata )
print "INFO: Waiting for headnode bootup ..."
driver._wait_until_running(node, wait_period=5, timeout=240)
print "INFO: Headnode active!"
if node.public_ips:
print '-'*51
print '{0:20} | {1:10} | {2:15}'.format(node.name, NODESTATES[node.state], node.public_ips[0])
print '-'*51
f = open('./PUBLIC_ADDRESS', 'w')
f.write(str(node.public_ips[0]))
f.close()
def start_worker(driver, configs, worker_names):
nodes = driver.list_nodes()
headnode = False
for node in nodes:
if node.name == "headnode" and node.state == NodeState.RUNNING:
headnode = node
if not headnode :
print "WARNING : No active headnode found"
return -1
# Setup userdata
userdata = configurator.getstring("worker")
userdata = userdata.replace("SET_HEADNODE_IP", headnode.public_ips[0])
logging.debug("Worker userdata : %s", userdata)
list_nodes = []
for worker_name in worker_names:
size = NodeSize(id=configs['WORKER_MACHINE_TYPE'], name=worker_name,
ram=None, disk=None, bandwidth=None, price=None, driver=driver)
image = NodeImage(id=configs['WORKER_IMAGE'], name=None, driver=driver)
node = driver.create_node(name=worker_name,
image=image,
size=size,
ex_keyname=configs['AWS_KEYPAIR_NAME'],
ex_securitygroup=configs['SECURITY_GROUP'],
ex_userdata=userdata )
list_nodes.append(node)
logging.info("Worker node started : %s",str(node))
def start_worker(driver, configs, worker_names):
nodes = driver.list_nodes()
headnode = False
for node in nodes:
if node.name == "headnode" and node.state == NodeState.RUNNING:
headnode = node
if not headnode:
print "WARNING : No active headnode found"
return -1
# Setup userdata
userdata = configurator.getstring("worker")
userdata = userdata.replace("SET_HEADNODE_IP", headnode.public_ips[0])
logging.debug("Worker userdata : %s", userdata)
list_nodes = []
for worker_name in worker_names:
size = NodeSize(id=configs['WORKER_MACHINE_TYPE'],
name=worker_name,
ram=None,
disk=None,
bandwidth=None,
price=None,
driver=driver)
image = NodeImage(id=configs['WORKER_IMAGE'], name=None, driver=driver)
node = driver.create_node(name=worker_name,
image=image,
size=size,
ex_keyname=configs['AWS_KEYPAIR_NAME'],
ex_securitygroup=configs['SECURITY_GROUP'],
ex_userdata=userdata)
list_nodes.append(node)
logging.info("Worker node started : %s", str(node))
def start_headnode(driver, configs):
userdata = configurator.getstring("headnode")
# Check if headnode
nodes = driver.list_nodes()
headnode = False
for node in nodes:
if node.name == "headnode" and node.state == NodeState.RUNNING:
headnode = node
print "INFO: Headnode is RUNNING"
return 0
sizes = driver.list_sizes()
size = [ s for s in sizes if s.id == configs['HEADNODE_MACHINE_TYPE'] ]
if not size:
logging.info("ec2headnodeimage not legal/valid : %s", configs['HEADNODE_MACHINE_TYPE'])
sys.stderr.write("HEADNODE_MACHINE_TYPE not legal/valid \n")
exit(-1);
image = NodeImage(id=configs['HEADNODE_IMAGE'], name=None, driver=driver)
node = driver.create_node(name='headnode',
image=image,
size=size[0],
ex_keyname=configs['AWS_KEYPAIR_NAME'],
ex_securitygroup=configs['SECURITY_GROUP'],
ex_userdata=userdata )
print "INFO: Waiting for headnode bootup ..."
driver._wait_until_running(node, wait_period=5, timeout=240)
print "INFO: Headnode active!"
if node.public_ips:
print '-'*51
print '{0:20} | {1:10} | {2:15}'.format(node.name, NODESTATES[node.state], node.public_ips[0])
print '-'*51
f = open('./PUBLIC_ADDRESS', 'w')
f.write(str(node.public_ips[0]))
f.close()
def start_headnode(driver, configs):
userdata = configurator.getstring("headnode")
# Check if headnode
nodes = driver.list_nodes()
headnode = False
for node in nodes:
if node.name == "headnode" and node.state == NodeState.RUNNING:
headnode = node
print "INFO: Headnode is RUNNING"
return 0
size = NodeSize(id=configs['HEADNODE_MACHINE_TYPE'],
name='headnode',
ram=None,
disk=None,
bandwidth=None,
price=None,
driver=driver)
image = NodeImage(id=configs['HEADNODE_IMAGE'], name=None, driver=driver)
node = driver.create_node(name='headnode',
image=image,
size=size,
ex_keyname=configs['AWS_KEYPAIR_NAME'],
ex_securitygroup=configs['SECURITY_GROUP'],
ex_userdata=userdata)
print "INFO: Waiting for headnode bootup ..."
driver._wait_until_running(node, wait_period=5, timeout=240)
print "INFO: Headnode active!"
if node.public_ips:
print '-' * 51
print '{0:20} | {1:10} | {2:15}'.format(node.name,
NODESTATES[node.state],
node.public_ips[0])
print '-' * 51
f = open('./PUBLIC_ADDRESS', 'w')
f.write(str(node.public_ips[0]))
f.close()
def start_worker(driver, configs, worker_names):
nodes = driver.list_nodes()
headnode = False
disk_info = False
volume = False
disk_info = ""
concurrency = "-c 1" # Default concurrency for workers
init_string = ""
for node in nodes:
if node.name == "headnode" and node.state == NodeState.RUNNING:
headnode = node
if not headnode :
print "WARNING : No active headnode found"
return -1
# Setup userdata
userdata = configurator.getstring("worker")
userdata = userdata.replace("SET_HEADNODE_IP", headnode.public_ips[0])
# Setting worker concurrency level
if 'WORKER_CONCURRENCY' in configs:
concurrency = "-c " + str(configs['WORKER_CONCURRENCY'])
userdata = userdata.replace("SET_CONCURRENCY", concurrency)
# Inserting WORKER_INIT_SCRIPT
if 'WORKER_INIT_SCRIPT' in configs :
if not os.path.isfile(configs['WORKER_INIT_SCRIPT']):
print "Unable to read WORKER_INIT_SCRIPT"
exit(-1);
init_string = open(configs['WORKER_INIT_SCRIPT'], 'r').read()
userdata = userdata.replace("#WORKER_INIT_SCRIPT", init_string)
if 'WORKER_DISK' in configs:
disk_info = configs['WORKER_DISK'].split(" ")
location = driver.list_locations()[0];
volume = driver.create_volume(size=500, location=location, name="test volume 500gb")
else:
print "No worker_disk defined"
logging.info("Worker userdata : %s", userdata)
list_nodes = []
sizes = driver.list_sizes()
size = [ s for s in sizes if s.id == configs['WORKER_MACHINE_TYPE'] ]
if not size:
logging.info("ec2workerimage not legal/valid : %s", configs['WORKER_MACHINE_TYPE'])
sys.stderr.write("WORKER_MACHINE_TYPE not legal/valid \n")
exit(-1);
image = NodeImage(id=configs['WORKER_IMAGE'], name=None, driver=driver)
print size
for worker_name in worker_names:
node = driver.create_node(name=worker_name,
image=image,
size=size[0],
ex_keyname=configs['AWS_KEYPAIR_NAME'],
ex_securitygroup=configs['SECURITY_GROUP'],
#block_device_mapping=mapping,
ex_userdata=userdata )
if volume:
driver.attach_volume(node, volume, device=disk_info[1] )
list_nodes.append(node)
logging.info("Worker node started : %s",str(node))
