def __init__(self, host, port, hadoop_version=YARN_PROTOCOL_VERSION):
'''
:param host: Hostname or IP address of the ResourceManager
:type host: string
:param port: RPC Port of the ResourceManager
:type port: int
:param hadoop_version: What hadoop protocol version should be used (default: 9)
:type hadoop_version: int
'''
if hadoop_version < 9:
raise Exception("Only protocol versions >= 9 supported")
context_proto = "org.apache.hadoop.yarn.api.ApplicationClientProtocolPB"
self.host = host
self.port = port
self.service_stub_class = application_client_protocol.ApplicationClientProtocolService_Stub
self.service = RpcService(self.service_stub_class, context_proto,
self.port, self.host, hadoop_version)
log.debug("Created client for %s:%s", host, port)
def __init__(self, host, port, hadoop_version=YARN_PROTOCOL_VERSION):
'''
:param host: Hostname or IP address of the ResourceManager
:type host: string
:param port: RPC Port of the ResourceManager
:type port: int
:param hadoop_version: What hadoop protocol version should be used (default: 9)
:type hadoop_version: int
'''
if hadoop_version < 9:
raise Exception("Only protocol versions >= 9 supported")
context_proto = "org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB"
self.host = host
self.port = port
self.service_stub_class = application_master_protocol.ApplicationMasterProtocolService_Stub
self.service = RpcService(self.service_stub_class, context_proto, self.port, self.host, hadoop_version)
log.debug("Created Master for %s:%s", host, port)
class YarnAppMaster(object):
"""
A pure python Yarn Master
"""
def __init__(self, host, port, hadoop_version=YARN_PROTOCOL_VERSION):
'''
:param host: Hostname or IP address of the ResourceManager
:type host: string
:param port: RPC Port of the ResourceManager
:type port: int
:param hadoop_version: What hadoop protocol version should be used (default: 9)
:type hadoop_version: int
'''
if hadoop_version < 9:
raise Exception("Only protocol versions >= 9 supported")
context_proto = "org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB"
self.host = host
self.port = port
self.service_stub_class = application_master_protocol.ApplicationMasterProtocolService_Stub
self.service = RpcService(self.service_stub_class, context_proto, self.port, self.host, hadoop_version)
log.debug("Created Master for %s:%s", host, port)
def submit(self, client):
clc = self._launchcontext()
prio = dict(priority=YARN_CONTAINER_PRIORITY_DEFAULT)
res = dict(memory=YARN_CONTAINER_MEM_DEFAULT, virtual_cores=YARN_CONTAINER_CPU_DEFAULT)
# appid, maxmem, maxcores = _new_app(client)
newapp = client.get_new_application()
newapp_dict = pb_to_dict(newapp)
appid = newapp_dict['application_id']
maxmem, maxcores = newapp_dict['maximumCapability']['memory'], newapp_dict['maximumCapability']['virtual_cores']
# name='cluster_timestamp', full_name='hadoop.yarn.ApplicationIdProto.cluster_timestamp', index=1,
appData = {'application_id': appid,
'application_name': "YarnBZApp",
'queue': 'default',
'priority': 1,
'unmanaged_am': True, # what does this mean?
'am_container_spec': clc,
'resource': res,
'maxAppAttempts': 2,
'applicationType': "PYTHON",
'keep_containers_across_application_attempts': False,
}
client.submit_application(**appData)
app = pb_to_dict(client.get_application_report(appid['cluster_timestamp'], appid['id']))
import ipdb
ipdb.set_trace()
# keep the am_rm token
tok = app['application_report']['am_rm_token']
channel = self.service.channel
#TODO: better data structure like ugi
channel.token = tok
channel.appid = appid
# current point of investigation
#add_token(channel.ugi, token_alias(channel), tok) Store token in external datastructure for later use
#
# # register the unmanaged appmaster
# unmanagedappmaster.managed = false (AM is not on cluster)
# register(unmanagedappmaster)
self.register(app)
# wait_for_attempt_state(app, Int32(1), YarnApplicationAttemptStateProto.APP_ATTEMPT_RUNNING) || throw(YarnException("Application attempt could not be launched"))
#
# # initialize complete node list for appmaster
# nodes(client; nodelist=unmanagedappmaster.nodes)
#
# # start the am_rm processing task once the app attempt starts running if it is an unmanaged application master
# @async(process_am_rm(unmanagedappmaster))
# app
# end
def _launchcontext(self, command="", environment={}, service_data={}):
c = dict(
command=command,
environment=[yarn_protos.StringStringMapProto(key=k, value=v) for k, v in environment.iteritems()],
service_data=[yarn_protos.StringBytesMapProto(key=k, value=v) for k, v in service_data.iteritems()],
)
return c
def register(self, app):
#app = pb_to_dict(client.get_application_report(appid['cluster_timestamp'], appid['id']))
app_report = app['application_report']
data = dict(host=self.host, rpc_port=self.port, tracking_url=app_report['trackingUrl'])
req = dict_to_pb(yarn_service_protos.RegisterApplicationMasterRequestProto, data)
self.service.registerApplicationMaster(req)
resource = yarn_protos.ResourceRequestProto()
resource.priority.priority = 1
resource.num_containers = 1
resource.resource_name = "*" #Any node will do
resource.capability.memory = 1
resource.capability.virtual_cores = 1
request = yarn_service_protos.AllocateRequestProto()
request.ask.extend([resource])
#req = dict_to_pb(yarn_service_protos.AllocateRequestProto, data)
self.service.allocate(request)
class Client(object):
"""
A pure python Yarn client. Yarn clients are used for submitting jobs to a Yarn ResourceManager
"""
def __init__(self, host, port, hadoop_version=YARN_PROTOCOL_VERSION):
'''
:param host: Hostname or IP address of the ResourceManager
:type host: string
:param port: RPC Port of the ResourceManager
:type port: int
:param hadoop_version: What hadoop protocol version should be used (default: 9)
:type hadoop_version: int
'''
if hadoop_version < 9:
raise Exception("Only protocol versions >= 9 supported")
context_proto = "org.apache.hadoop.yarn.api.ApplicationClientProtocolPB"
self.host = host
self.port = port
self.service_stub_class = application_client_protocol.ApplicationClientProtocolService_Stub
self.service = RpcService(self.service_stub_class, context_proto,
self.port, self.host, hadoop_version)
log.debug("Created client for %s:%s", host, port)
def get_applications(self, app_states=None):
ALL_APP_STATES = [
yarn_protos.ACCEPTED, yarn_protos.NEW, yarn_protos.NEW_SAVING,
yarn_protos.SUBMITTED, yarn_protos.RUNNING, yarn_protos.FINISHED,
yarn_protos.KILLED, yarn_protos.FAILED
]
req = yarn_service_protos.GetApplicationsRequestProto()
if not app_states:
app_states = ALL_APP_STATES
elif type(app_states) in (tuple, list):
pass
else:
app_states = [app_states]
for app_state in app_states:
req.application_states.append(app_state)
response = self.service.getApplications(req)
if response:
return response.applications
else:
return []
def get_application_report(self, cluster_timestamp, app_id):
"""
:type cluster_timestamp: long
:type app_id: int
"""
req = yarn_service_protos.GetApplicationReportRequestProto()
req.application_id.id = int(app_id)
req.application_id.cluster_timestamp = cluster_timestamp
return self.service.getApplicationReport(req)
def submit_application(self, **kwargs):
req = dict_to_pb(yarn_service_protos.SubmitApplicationRequestProto,
kwargs)
return self.service.submitApplication(req)
def get_new_application(self):
req = yarn_service_protos.GetNewApplicationRequestProto()
return self.service.getNewApplication(req)
def get_cluster_nodes(self):
req = yarn_service_protos.GetClusterNodesRequestProto()
response = self.service.getClusterNodes(req)
if response:
return response.nodeReports
else:
return []
def get_cluster_metrics(self):
req = yarn_service_protos.GetClusterMetricsRequestProto()
resp = self.service.getClusterMetrics(req)
return resp
def get_queue_info(self, queue='root'):
req = yarn_service_protos.GetQueueInfoRequestProto()
req.queueName = queue
return self.service.getQueueInfo(req)
def get_queue_user_acls(self):
req = yarn_service_protos.GetQueueInfoRequestProto()
return self.service.getQueueUserAcls(req)
def get_containers(self, cluster_timestamp, app_id):
req = yarn_service_protos.GetContainersRequestProto()
req.application_attempt_id.application_id.id = app_id
req.application_attempt_id.application_id.cluster_timestamp = cluster_timestamp
return self.service.getContainers(req)
def get_container_report(self, cluster_timestamp, app_id, container_id):
req = yarn_service_protos.GetContainerReportRequestProto()
req.container_id.id = container_id
req.container_id.application_id.id = app_id
req.container_id.application_id.cluster_timestamp
return self.service.getContainerReport(req)
def force_kill_application(self, cluster_timestamp, app_id):
req = yarn_service_protos.KillApplicationRequestProto()
req.application_id.id = app_id
req.application_id.cluster_timestamp = cluster_timestamp
return self.service.forceKillApplication(req)
def move_application_across_queues(self, cluster_timestamp, app_id,
target_queue):
req = yarn_service_protos.MoveApplicationAcrossQueuesRequestProto(
application_id=yarn_service_protos.ApplicationIdProto(
id=app_id, cluster_timestamp=cluster_timestamp),
target_queue=target_queue)
return self.service.moveApplicationAcrossQueue(req)
def get_application_attempt_report(self, cluster_timestamp, app_id):
req = yarn_service_protos.GetApplicationAttemptReportRequestProto(
application_id=yarn_protos.ApplicationIdProto(
id=app_id, cluster_timestamp=cluster_timestamp))
return self.service.getApplicationAttemptReport(req)
def get_application_attempts(self, cluster_timestamp, app_id):
req = yarn_service_protos.GetApplicationAttemptsRequestProto(
application_id=yarn_protos.ApplicationIdProto(
id=app_id, cluster_timestamp=cluster_timestamp))
return self.service.getApplicationAttempts(req)
def get_delegation_token(self):
pass
def renew_delegation_token(self):
pass
def cancel_delegation_token(self):
pass
class Client(object):
"""
A pure python Yarn client. Yarn clients are used for submitting jobs to a Yarn ResourceManager
"""
def __init__(self, host, port, hadoop_version=YARN_PROTOCOL_VERSION):
'''
:param host: Hostname or IP address of the ResourceManager
:type host: string
:param port: RPC Port of the ResourceManager
:type port: int
:param hadoop_version: What hadoop protocol version should be used (default: 9)
:type hadoop_version: int
'''
if hadoop_version < 9:
raise Exception("Only protocol versions >= 9 supported")
context_proto = "org.apache.hadoop.yarn.api.ApplicationClientProtocolPB"
self.host = host
self.port = port
self.service_stub_class = application_client_protocol.ApplicationClientProtocolService_Stub
self.service = RpcService(self.service_stub_class, context_proto, self.port, self.host, hadoop_version)
log.debug("Created client for %s:%s", host, port)
def get_applications(self, app_states=None):
ALL_APP_STATES = [
yarn_protos.ACCEPTED,
yarn_protos.NEW,
yarn_protos.NEW_SAVING,
yarn_protos.SUBMITTED,
yarn_protos.RUNNING,
yarn_protos.FINISHED,
yarn_protos.KILLED,
yarn_protos.FAILED
]
req = yarn_service_protos.GetApplicationsRequestProto()
if not app_states:
app_states = ALL_APP_STATES
elif type(app_states) in (tuple, list):
pass
else:
app_states = [app_states]
for app_state in app_states:
req.application_states.append(app_state)
response = self.service.getApplications(req)
if response:
return response.applications
else:
return []
def get_application_report(self, cluster_timestamp, app_id):
"""
:type cluster_timestamp: long
:type app_id: int
"""
req = yarn_service_protos.GetApplicationReportRequestProto()
req.application_id.id = int(app_id)
req.application_id.cluster_timestamp = cluster_timestamp
return self.service.getApplicationReport(req)
def submit_application(self, **kwargs):
req = dict_to_pb(yarn_service_protos.SubmitApplicationRequestProto, kwargs)
return self.service.submitApplication(req)
def get_new_application(self):
req = yarn_service_protos.GetNewApplicationRequestProto()
return self.service.getNewApplication(req)
def get_cluster_nodes(self):
req = yarn_service_protos.GetClusterNodesRequestProto()
response = self.service.getClusterNodes(req)
if response:
return response.nodeReports
else:
return []
def get_cluster_metrics(self):
req = yarn_service_protos.GetClusterMetricsRequestProto()
resp = self.service.getClusterMetrics(req)
return resp
def get_queue_info(self, queue='root'):
req = yarn_service_protos.GetQueueInfoRequestProto()
req.queueName = queue
return self.service.getQueueInfo(req)
def get_queue_user_acls(self):
req = yarn_service_protos.GetQueueInfoRequestProto()
return self.service.getQueueUserAcls(req)
def get_containers(self, cluster_timestamp, app_id):
req = yarn_service_protos.GetContainersRequestProto()
req.application_attempt_id.application_id.id = app_id
req.application_attempt_id.application_id.cluster_timestamp = cluster_timestamp
return self.service.getContainers(req)
def get_container_report(self, cluster_timestamp, app_id, container_id):
req = yarn_service_protos.GetContainerReportRequestProto()
req.container_id.id = container_id
req.container_id.application_id.id = app_id
req.container_id.application_id.cluster_timestamp
return self.service.getContainerReport(req)
def force_kill_application(self, cluster_timestamp, app_id):
req = yarn_service_protos.KillApplicationRequestProto()
req.application_id.id = app_id
req.application_id.cluster_timestamp = cluster_timestamp
return self.service.forceKillApplication(req)
def move_application_across_queues(self, cluster_timestamp, app_id, target_queue):
req = yarn_service_protos.MoveApplicationAcrossQueuesRequestProto(
application_id=yarn_service_protos.ApplicationIdProto(
id=app_id,
cluster_timestamp=cluster_timestamp
),
target_queue=target_queue
)
return self.service.moveApplicationAcrossQueue(req)
def get_application_attempt_report(self, cluster_timestamp, app_id):
req = yarn_service_protos.GetApplicationAttemptReportRequestProto(
application_id=yarn_protos.ApplicationIdProto(
id=app_id,
cluster_timestamp=cluster_timestamp
)
)
return self.service.getApplicationAttemptReport(req)
def get_application_attempts(self, cluster_timestamp, app_id):
req = yarn_service_protos.GetApplicationAttemptsRequestProto(
application_id=yarn_protos.ApplicationIdProto(
id=app_id,
cluster_timestamp=cluster_timestamp
)
)
return self.service.getApplicationAttempts(req)
def get_delegation_token(self):
pass
def renew_delegation_token(self):
pass
def cancel_delegation_token(self):
pass
