def _stop_server_all( self, req_d):
ls = sys_def_mdl.get_system_defaults('local_settings', 'Master')
branch = ls['branch']
self.app.logger.info("Stopping allworkers for the %s branch" % (
branch ))
workers = wkr.get_active_workers(branch)
workers = [json_prep( worker ) for worker in workers]
launcher_config = sys_def_mdl.get_system_defaults(
setting_name = 'launcher_config', component='Master' )
conn = boto.sqs.connect_to_region('us-east-1')
lq = conn.create_queue( launcher_config['launcher_sqs_in'] )
worker_list = []
active_statuses = [wkr.READY, wkr.RUNNING]
for worker in workers:
if worker['status'] in active_statuses:
launcher_message = {'action': 'stop-server',
'worker_id': worker['worker_id'],
}
worker_list.append( worker['worker_id'] )
mess = Message(body=json.dumps( launcher_message ))
lq.write( mess )
msg = {'status': 'complete',
'data': [w['worker_id'] for w in workers] }
status = 200
return ( msg, status )
def _terminate_single_worker( self, worker_id):
worker = wkr.get_ANWorker( worker_id=worker_id )
self.app.logger.info("%r" % worker )
if worker['status'] in [wkr.CONFIG, wkr.NA]:
worker = wkr.update_ANWorker( worker_id,
status=wkr.TERMINATED)
msg = {'status':'complete',
'data' : json_prep( worker )}
status = 200
elif wkr.confirm_worker_running( worker ):
#we have an active cluster
master = mstr.get_active_master()
if master:
launcher_message = {'action':'terminate',
'worker_id': worker_id}
launcher_config = sys_def_mdl.get_system_defaults(
setting_name = 'launcher_config', component='Master' )
conn = boto.sqs.connect_to_region('us-east-1')
lq = conn.create_queue( launcher_config['launcher_sqs_in'] )
worker = wkr.update_ANWorker( worker_id,
status=wkr.MARKED_FOR_TERMINATION)
mess = Message(body=json.dumps( launcher_message ))
lq.write( mess )
msg = {'status':'complete',
'data' : json_prep( worker ) }
status = 200
else:
msg = {'status': 'error',
'data' : {'worker_id': worker_id},
'message' : 'Running Cluster without an active master'
}
status = 409 #Conflict
else:
worker = wkr.update_ANWorker( worker_id,
status=wkr.TERMINATED_WITH_ERROR)
msg = {'status':'complete',
'data' : json_prep( worker )}
status = 200
return (msg, status)
def GET( self, request):
if self.worker_id is None:
#return active workers
if request.args.get('branch'):
branch = None
else:
ls = sys_def_mdl.get_system_defaults('local_settings', 'Master')
branch = ls['branch']
self.app.logger.info("GETting workers for the %s branch" % (
branch ))
workers = wkr.get_active_workers(branch)
workers = [json_prep( worker ) for worker in workers]
if workers:
msg = {
'status' : 'complete',
'data' : workers
}
status = 200
else:
msg = {
'status' : 'error',
'data' : [],
'message': 'No workers available'
}
status = 404
else:
result = wkr.get_ANWorker( worker_id=self.worker_id )
if result:
msg = {'status' : 'complete',
'data' : json_prep( result )
}
status = 200
else:
msg = {'status': 'error',
'data' : {'worker_id' : self.worker_id},
'message' : 'Worker not found'
}
status = 404
return ( msg, status )
def _activate_worker_all(self, req_d ):
ls = sys_def_mdl.get_system_defaults('local_settings', 'Master')
branch = ls['branch']
self.app.logger.info("GETting workers for the %s branch" % (
branch ))
workers = wkr.get_active_workers(branch)
workers = [json_prep( worker ) for worker in workers]
worker_list = []
for worker in workers:
if worker['status'] == 0:
self._activate_single_worker( worker['worker_id'] )
worker_list.append( worker['worker_id'] )
msg = {'status':'complete',
'data' : worker_list }
status = 200
return ( msg, status )
def _terminate_worker_all( self, req_d):
ls = sys_def_mdl.get_system_defaults('local_settings', 'Master')
branch = ls['branch']
self.app.logger.info("GETting workers for the %s branch" % (
branch ))
workers = wkr.get_active_workers(branch)
workers = [json_prep( worker ) for worker in workers]
terminatable_statuses = [wkr.NA, wkr.CONFIG, wkr.READY, wkr.RUNNING,
wkr.MARKED_FOR_TERMINATION]
for worker in workers:
if worker['status'] in terminatable_statuses:
self._terminate_single_worker( worker['worker_id'] )
worker_list = [worker['worker_id'] for worker in workers]
msg = {'status':'complete',
'data' : worker_list }
status = 200
return ( msg, status )
def _init_worker( self, req_d):
msg = {'status': 'error',
'data' : '',
'message': 'Unable to initialize worker'}
status = 404
result = None
if( req_d['master_name'] and req_d['master_name'] != "None"):
result = wkr.get_ANWorkerBase( req_d['cluster_type'], req_d['aws_region'] )
if result:
new_worker_settings = {
'master_name': req_d['master_name'],
'cluster_type': result['cluster_type'],
'aws_region' : result['aws_region'],
'cluster_name' : self._gen_name( req_d['master_name'],
result['prefix'] ),
'num_nodes': 0,
'status':wkr.CONFIG,
}
starcluster_config = {
'cluster_name': new_worker_settings['cluster_name'],
'spot_bid': result['spot_bid'],
'key_name' : 'SET BY MASTER',
'key_location': 'SET BY MASTER',
'iam_profile': result['iam_profile'],
'force_spot_master': result['force_spot_master'],
'cluster_size': result['cluster_size'],
'plugins': result['plugins'],
'node_instance_type': result['instance_type'],
'node_image_id': result['image_id'],
'aws_region': result['aws_region']
}
new_worker = None
try:
new_worker = wkr.insert_ANWorker(starcluster_config=starcluster_config,
**new_worker_settings)
except Exception as e:
self.app.logger.error("Attempted to create [%r] [%r]" % (
new_worker_settings, starcluster_config ))
self.app.logger.error("Received exception [%r]" % (e))
pass
if new_worker:
msg = {'status' : 'complete',
'data' : json_prep( new_worker )}
status=200
return (msg, status)
def get_master():
"""
API
Returns currently active master instance
"""
current_app.logger.info('get_master')
import masterdirac.models.master as mstr
import masterdirac.models.systemdefaults as sys_def
local_settings = sys_def.get_system_defaults('local_settings',
'Master')
master = mstr.get_active_master( local_settings['branch'] )
if master is not None:
msg = {'status' : 'complete',
'data' : json_prep(master) }
status = 200
else:
msg = {'status' : 'error',
'data' : '',
'message':'No Active Master'}
status = 404
return Response( json.dumps( msg ), mimetype='application/json',
status = status )
def GET(self):
launcher_config = sys_def.get_system_defaults(setting_name="launcher_config", component="Master")
message_q = launcher_config["launcher_sqs_out"]
conn = boto.sqs.connect_to_region("us-east-1")
q = conn.create_queue(message_q)
messages = q.get_messages(10)
outbound = []
while messages:
for message in messages:
message_body = message.get_body()
message_dict = json.loads(message_body)
self.logger.info("Message Rec'd [%s]" % message_body)
if "recipient" not in message_dict:
# garbage message
q.delete_message(message)
elif message_dict["recipient"] == "console":
# message for this element
if not self._dup(message_dict["message"]):
outbound.append(message_dict["message"])
q.delete_message(message)
messages = q.get_messages(10)
msg = {"status": "complete", "data": [json_prep(m) for m in outbound]}
status = 200
return (msg, status)
