def _stop_server_all( self, req_d): ls = sys_def_mdl.get_system_defaults('local_settings', 'Master') branch = ls['branch'] self.app.logger.info("Stopping allworkers for the %s branch" % ( branch )) workers = wkr.get_active_workers(branch) workers = [json_prep( worker ) for worker in workers] launcher_config = sys_def_mdl.get_system_defaults( setting_name = 'launcher_config', component='Master' ) conn = boto.sqs.connect_to_region('us-east-1') lq = conn.create_queue( launcher_config['launcher_sqs_in'] ) worker_list = [] active_statuses = [wkr.READY, wkr.RUNNING] for worker in workers: if worker['status'] in active_statuses: launcher_message = {'action': 'stop-server', 'worker_id': worker['worker_id'], } worker_list.append( worker['worker_id'] ) mess = Message(body=json.dumps( launcher_message )) lq.write( mess ) msg = {'status': 'complete', 'data': [w['worker_id'] for w in workers] } status = 200 return ( msg, status )
def _terminate_single_worker( self, worker_id): worker = wkr.get_ANWorker( worker_id=worker_id ) self.app.logger.info("%r" % worker ) if worker['status'] in [wkr.CONFIG, wkr.NA]: worker = wkr.update_ANWorker( worker_id, status=wkr.TERMINATED) msg = {'status':'complete', 'data' : json_prep( worker )} status = 200 elif wkr.confirm_worker_running( worker ): #we have an active cluster master = mstr.get_active_master() if master: launcher_message = {'action':'terminate', 'worker_id': worker_id} launcher_config = sys_def_mdl.get_system_defaults( setting_name = 'launcher_config', component='Master' ) conn = boto.sqs.connect_to_region('us-east-1') lq = conn.create_queue( launcher_config['launcher_sqs_in'] ) worker = wkr.update_ANWorker( worker_id, status=wkr.MARKED_FOR_TERMINATION) mess = Message(body=json.dumps( launcher_message )) lq.write( mess ) msg = {'status':'complete', 'data' : json_prep( worker ) } status = 200 else: msg = {'status': 'error', 'data' : {'worker_id': worker_id}, 'message' : 'Running Cluster without an active master' } status = 409 #Conflict else: worker = wkr.update_ANWorker( worker_id, status=wkr.TERMINATED_WITH_ERROR) msg = {'status':'complete', 'data' : json_prep( worker )} status = 200 return (msg, status)
def GET( self, request): if self.worker_id is None: #return active workers if request.args.get('branch'): branch = None else: ls = sys_def_mdl.get_system_defaults('local_settings', 'Master') branch = ls['branch'] self.app.logger.info("GETting workers for the %s branch" % ( branch )) workers = wkr.get_active_workers(branch) workers = [json_prep( worker ) for worker in workers] if workers: msg = { 'status' : 'complete', 'data' : workers } status = 200 else: msg = { 'status' : 'error', 'data' : [], 'message': 'No workers available' } status = 404 else: result = wkr.get_ANWorker( worker_id=self.worker_id ) if result: msg = {'status' : 'complete', 'data' : json_prep( result ) } status = 200 else: msg = {'status': 'error', 'data' : {'worker_id' : self.worker_id}, 'message' : 'Worker not found' } status = 404 return ( msg, status )
def _activate_worker_all(self, req_d ): ls = sys_def_mdl.get_system_defaults('local_settings', 'Master') branch = ls['branch'] self.app.logger.info("GETting workers for the %s branch" % ( branch )) workers = wkr.get_active_workers(branch) workers = [json_prep( worker ) for worker in workers] worker_list = [] for worker in workers: if worker['status'] == 0: self._activate_single_worker( worker['worker_id'] ) worker_list.append( worker['worker_id'] ) msg = {'status':'complete', 'data' : worker_list } status = 200 return ( msg, status )
def _terminate_worker_all( self, req_d): ls = sys_def_mdl.get_system_defaults('local_settings', 'Master') branch = ls['branch'] self.app.logger.info("GETting workers for the %s branch" % ( branch )) workers = wkr.get_active_workers(branch) workers = [json_prep( worker ) for worker in workers] terminatable_statuses = [wkr.NA, wkr.CONFIG, wkr.READY, wkr.RUNNING, wkr.MARKED_FOR_TERMINATION] for worker in workers: if worker['status'] in terminatable_statuses: self._terminate_single_worker( worker['worker_id'] ) worker_list = [worker['worker_id'] for worker in workers] msg = {'status':'complete', 'data' : worker_list } status = 200 return ( msg, status )
def _init_worker( self, req_d): msg = {'status': 'error', 'data' : '', 'message': 'Unable to initialize worker'} status = 404 result = None if( req_d['master_name'] and req_d['master_name'] != "None"): result = wkr.get_ANWorkerBase( req_d['cluster_type'], req_d['aws_region'] ) if result: new_worker_settings = { 'master_name': req_d['master_name'], 'cluster_type': result['cluster_type'], 'aws_region' : result['aws_region'], 'cluster_name' : self._gen_name( req_d['master_name'], result['prefix'] ), 'num_nodes': 0, 'status':wkr.CONFIG, } starcluster_config = { 'cluster_name': new_worker_settings['cluster_name'], 'spot_bid': result['spot_bid'], 'key_name' : 'SET BY MASTER', 'key_location': 'SET BY MASTER', 'iam_profile': result['iam_profile'], 'force_spot_master': result['force_spot_master'], 'cluster_size': result['cluster_size'], 'plugins': result['plugins'], 'node_instance_type': result['instance_type'], 'node_image_id': result['image_id'], 'aws_region': result['aws_region'] } new_worker = None try: new_worker = wkr.insert_ANWorker(starcluster_config=starcluster_config, **new_worker_settings) except Exception as e: self.app.logger.error("Attempted to create [%r] [%r]" % ( new_worker_settings, starcluster_config )) self.app.logger.error("Received exception [%r]" % (e)) pass if new_worker: msg = {'status' : 'complete', 'data' : json_prep( new_worker )} status=200 return (msg, status)
def get_master(): """ API Returns currently active master instance """ current_app.logger.info('get_master') import masterdirac.models.master as mstr import masterdirac.models.systemdefaults as sys_def local_settings = sys_def.get_system_defaults('local_settings', 'Master') master = mstr.get_active_master( local_settings['branch'] ) if master is not None: msg = {'status' : 'complete', 'data' : json_prep(master) } status = 200 else: msg = {'status' : 'error', 'data' : '', 'message':'No Active Master'} status = 404 return Response( json.dumps( msg ), mimetype='application/json', status = status )
def GET(self): launcher_config = sys_def.get_system_defaults(setting_name="launcher_config", component="Master") message_q = launcher_config["launcher_sqs_out"] conn = boto.sqs.connect_to_region("us-east-1") q = conn.create_queue(message_q) messages = q.get_messages(10) outbound = [] while messages: for message in messages: message_body = message.get_body() message_dict = json.loads(message_body) self.logger.info("Message Rec'd [%s]" % message_body) if "recipient" not in message_dict: # garbage message q.delete_message(message) elif message_dict["recipient"] == "console": # message for this element if not self._dup(message_dict["message"]): outbound.append(message_dict["message"]) q.delete_message(message) messages = q.get_messages(10) msg = {"status": "complete", "data": [json_prep(m) for m in outbound]} status = 200 return (msg, status)