def create_reservation_requests_proto(reservation_resources=None,
interpreter=None):
if reservation_resources:
if type(reservation_resources) in (tuple, list):
for reservation in reservation_resources:
if not isinstance(reservation,
yarn_protos.ReservationRequestProto):
raise YarnError(
"reservation_resources need to be a list of ReservationRequestProto."
)
else:
if isinstance(reservation_resources,
yarn_protos.ReservationRequestProto):
reservation_resources = [reservation_resources]
else:
raise YarnError(
"reservation_resources need to be a list of ReservationRequestProto."
)
if interpreter:
if not isinstance(interpreter, RESERVATION_REQUEST_INTERPRETER):
raise YarnError(
"interpreter need to be of type Enum RESERVATION_REQUEST_INTERPRETER."
)
return yarn_protos.ReservationRequestsProto(
reservation_resources=reservation_resources, interpreter=interpreter)
def create_local_resource_proto(key=None,
scheme=None,
host=None,
port=None,
resource_file=None,
userInfo=None,
size=None,
timestamp=None,
recource_type=None,
visibility=None,
pattern=None):
resource = yarn_protos.URLProto(scheme=scheme,
host=host,
port=port,
file=resource_file,
userInfo=userInfo)
if recource_type:
if not isinstance(recource_type, LOCAL_RESOURCE_TYPE):
raise YarnError(
"recource_type need to be of type LOCAL_RESOURCE_TYPE.")
if visibility:
if not isinstance(visibility, LOCAL_RESOURCE_VISIBILITY):
raise YarnError(
"visibility need to be of type LOCAL_RESOURCE_VISIBILITY.")
local_resource = yarn_protos.LocalResourceProto(resource=resource,
size=size,
timestamp=timestamp,
type=recource_type,
visibility=visibility,
pattern=pattern)
return yarn_protos.StringLocalResourceMapProto(key=key,
value=local_resource)
def create_container_resource_increase_request_proto(container_id=None,
capability=None):
if capability:
if not isinstance(capability, yarn_protos.ResourceProto):
raise YarnError("capability need to be of type ResourceProto.")
if container_id:
if not isinstance(container_id, yarn_protos.ContainerIdProto):
raise YarnError(
"container_id need to be of type ContainerIdProto.")
return yarn_protos.ContainerResourceIncreaseRequestProto(
container_id=container_id, capability=capability)
def create_taskid_proto(task_id, task_type=None, job_id=None):
if job_id:
if not isinstance(job_id, mr_protos.JobIdProto):
raise YarnError("job_id need to be of type JobIdProto.")
if task_type:
if not isinstance(task_type, TASKTYPE):
raise YarnError("task_type need to be of type TASKTYPE.")
return mr_protos.TaskIdProto(id=task_id,
job_id=job_id,
task_type=task_type)
def create_start_container_request(container_launch_context,
container_token=None):
if container_launch_context:
if not isinstance(container_launch_context,
yarn_protos.ContainerLaunchContextProto):
raise YarnError(
"container_launch_context need to be of type ContainerLaunchContextProto."
)
if container_token:
if not isinstance(container_token, security_protocol.TokenProto):
raise YarnError("container_token need to be of type TokenProto.")
return yarn_service_protos.StartContainerRequestProto(
container_launch_context=container_launch_context,
container_token=container_token)
def create_reservation_request_proto(capability=None,
num_containers=None,
concurrency=None,
duration=None):
if capability:
if not isinstance(capability, yarn_protos.ResourceProto):
raise YarnError("capability need to be of type ResourceProto.")
return yarn_protos.ReservationRequestProto(capability=capability,
num_containers=num_containers,
concurrency=concurrency,
duration=duration)
def __init__(self, path=None):
self.path = path or os.getenv('CERASTES_CONFIG', self.default_path)
if osp.exists(self.path):
try:
self.config = json.loads(open(self.path).read())
self.schema = json.loads(
resource_string(__name__, 'resources/config_schema.json'))
try:
js.validate(self.config, self.schema)
except js.ValidationError as e:
print e.message
except js.SchemaError as e:
print e
except Exception as e:
raise YarnError('Invalid configuration file %r.', self.path)
_logger.info('Instantiated configuration from %r.', self.path)
else:
raise YarnError('Invalid configuration file %r.', self.path)
def get_client(self, cluster_name, client_class, **kwargs):
"""
:param cluster_name: The client to look up. If the cluster name does not
exist and exception will be raised.
:param kwargs: additional arguments can be used to overwrite or add some
parameters defined in the configuration file.
"""
for cluster in self.config['clusters']:
if cluster['name'] == cluster_name:
return client_class(YarnConfig(**cluster), **kwargs)
# the name does not exist
raise YarnError('Cluster %s is not defined in configuration file.' %
cluster_name)
def create_reservation_definition_proto(reservation_requests=None,
arrival=None,
deadline=None,
reservation_name=None):
if reservation_requests:
if not isinstance(reservation_requests,
yarn_protos.ReservationRequestsProto):
raise YarnError(
"reservation_requests need to be of type ReservationRequestsProto."
)
return yarn_protos.ReservationDefinitionProto(
reservation_requests=reservation_requests,
arrival=arrival,
deadline=deadline,
reservation_name=reservation_name)
def create_container_resource_request(priority, resource_name, capability,
num_containers, relax_locality,
node_label_expression):
priority_proto = yarn_protos.PriorityProto(priority=priority)
if capability:
if not isinstance(capability, yarn_protos.ResourceProto):
raise YarnError("capability need to be of type ResourceProto.")
return yarn_protos.ResourceRequestProto(
priority=priority_proto,
resource_name=resource_name,
capability=capability,
num_containers=num_containers,
relax_locality=relax_locality,
node_label_expression=node_label_expression)
def create_container_context_proto(local_resources_map=None,
tokens=None,
service_data_map=None,
environment_map=None,
commands=None,
application_ACLs=None):
if local_resources_map:
if type(local_resources_map) in (tuple, list):
for local_resource in local_resources_map:
if not isinstance(local_resource,
yarn_protos.StringLocalResourceMapProto):
raise YarnError(
"local_resources_map need to be a list of StringLocalResourceMapProto."
)
else:
if isinstance(local_resources_map,
yarn_protos.StringLocalResourceMapProto):
local_resources_map = [local_resources_map]
else:
raise YarnError(
"local_resources_map need to be a list of StringLocalResourceMapProto."
)
if tokens:
if type(tokens) in (tuple, list):
for token in tokens:
if not isinstance(token, bytes):
raise YarnError("tokens need to be a list of bytes.")
else:
if isinstance(tokens, bytes):
tokens = [tokens]
else:
raise YarnError("tokens need to be a list of bytes.")
if service_data_map:
if type(service_data_map) in (tuple, list):
for service_data in service_data_map:
if not isinstance(service_data,
yarn_protos.StringBytesMapProto):
raise YarnError(
"service_data_map need to be a list of StringBytesMapProto."
)
else:
if isinstance(service_data_map, yarn_protos.StringBytesMapProto):
service_data_map = [service_data_map]
else:
raise YarnError(
"service_data_map need to be a list of StringBytesMapProto."
)
if environment_map:
if type(environment_map) in (tuple, list):
for environment in environment_map:
if not isinstance(environment,
yarn_protos.StringStringMapProto):
raise YarnError(
"environment_map need to be a list of StringStringMapProto."
)
else:
if isinstance(environment_map, yarn_protos.StringStringMapProto):
environment_map = [environment_map]
else:
raise YarnError(
"environment_map need to be a list of StringStringMapProto."
)
if commands:
if type(commands) in (tuple, list):
for command in commands:
if not isinstance(command, str):
raise YarnError("commands need to be a list of str.")
else:
if isinstance(commands, str):
commands = [commands]
else:
raise YarnError("commands need to be a list of str.")
if application_ACLs:
if type(application_ACLs) in (tuple, list):
for acl in application_ACLs:
if not isinstance(acl, yarn_protos.ApplicationACLMapProto):
raise YarnError(
"application_ACLs need to be a list of ApplicationACLMapProto."
)
else:
if isinstance(application_ACLs,
yarn_protos.ApplicationACLMapProto):
application_ACLs = [application_ACLs]
else:
raise YarnError(
"application_ACLs need to be a list of ApplicationACLMapProto."
)
return yarn_protos.ContainerLaunchContextProto(
localResources=local_resources_map,
tokens=tokens,
service_data=service_data_map,
environment=environment_map,
command=commands,
application_ACLs=application_ACLs)
def create_application_acl_proto(accessType=None, acl=None):
if accessType:
if not isinstance(accessType, APPLICATION_ACCESS_TYPE):
raise YarnError(
"accessType need to be of type APPLICATION_ACCESS_TYPE.")
return yarn_protos.ApplicationACLMapProto(accessType=accessType, acl=acl)
def create_service_data_proto(key=None, value=None):
if value:
if not isinstance(value, bytes):
raise YarnError("value need to be of type bytes.")
return yarn_protos.StringBytesMapProto(key=key, value=value)
def create_task_attempt_id_proto(attempt_id, task_id=None):
if task_id:
if not isinstance(task_id, mr_protos.TaskIdProto):
raise YarnError("task_id need to be of type TaskIdProto.")
return mr_protos.TaskAttemptIdProto(id=attempt_id, task_id=task_id)