def start_job(self,
task_id,
output_location,
arguments,
job_name,
cpu_cores=1,
total_physical_memory=2 * 1024,
requested_queue='cscs_viz'):
'''Launch a job on through the platfrom task manager
Args:
task_id(string): The uuid of the task to be launched
output_location(string): Location that where this job saves its outputs
arguments: The arguments that are passed to the task, they must
match the expectations of the parameters
cpu_cores(int): number of requested cores
total_physical_memory(int): number of megabytes per node
requested_queue(string): Where the job is expected to be launched
Returns:
A dictionary with information to track the the launched job:
job_id(string): the unique identifier of the launched job
websocket(string): url to a websocket where all logging of the job can be listened to
Raises:
'''
arguments = JSONable.data_hierachy_pre_json(arguments)
body = sh.swagger_create_type(PostJobSchema.PostJobSchema, locals())
return sh.swagger_type_to_dict(self._job_api.PostJob(body))
def get_tasks(self, task_name=None, git_commit=None, git_repo=None):
'''get the tasks from the platform task manager
Args:
task_name(string): The name of the task to be registered
git_commit(string): The commit used when the task is checked out
git_repo(string): The repo that contains the task code
The tasks returned will be a combination matching all the above
requirements. The function can be called with no arguments to get
all tasks.
Returns:
A list of dictionaries matching the queries
'''
query_fields = {}
if task_name:
query_fields['task_name'] = task_name
if git_commit:
query_fields['git_commit'] = git_commit
if git_repo:
query_fields['git_repo'] = git_repo
tasks = self._task_api.GetTask(**query_fields)
tasks = [sh.swagger_type_to_dict(t) for t in tasks.tasks]
return tasks
def get_tasks(self, task_name=None, git_commit=None, git_repo=None):
'''get the tasks from the platform task manager
Args:
task_name(string): The name of the task to be registered
git_commit(string): The commit used when the task is checked out
git_repo(string): The repo that contains the task code
The tasks returned will be a combination matching all the above
requirements. The function can be called with no arguments to get
all tasks.
Returns:
A list of dictionaries matching the queries
'''
query_fields = {}
if task_name:
query_fields['task_name'] = task_name
if git_commit:
query_fields['git_commit'] = git_commit
if git_repo:
query_fields['git_repo'] = git_repo
tasks = self._task_api.GetTask(**query_fields)
tasks = [sh.swagger_type_to_dict(t) for t in tasks.tasks]
return tasks
def start_job(self, task_id, output_location, arguments, job_name,
cpu_cores=1, total_physical_memory=2 * 1024, requested_queue='cscs_viz'):
'''Launch a job on through the platfrom task manager
Args:
task_id(string): The uuid of the task to be launched
output_location(string): Location that where this job saves its outputs
arguments: The arguments that are passed to the task, they must
match the expectations of the parameters
cpu_cores(int): number of requested cores
total_physical_memory(int): number of megabytes per node
requested_queue(string): Where the job is expected to be launched
Returns:
A dictionary with information to track the the launched job:
job_id(string): the unique identifier of the launched job
websocket(string): url to a websocket where all logging of the job can be listened to
Raises:
'''
arguments = JSONable.data_hierachy_pre_json(arguments)
body = sh.swagger_create_type(PostJobSchema.PostJobSchema, locals())
return sh.swagger_type_to_dict(self._job_api.PostJob(body))
def get_job(self, job_id):
'''get a job by id
Args:
job_id(str): the job id
'''
job = self._job_api.GetJobArg(job_id)
return sh.swagger_type_to_dict(job)
def get_job(self, job_id):
'''get a job by id
Args:
job_id(str): the job id
'''
job = self._job_api.GetJobArg(job_id)
return sh.swagger_type_to_dict(job)
def get_jobs(self):
'''Get the jobs for the user
Returns:
List of dictionaries, containing the job information
'''
jobs = self._job_api.GetJob()
jobs = [sh.swagger_type_to_dict(j) for j in jobs.jobs]
return jobs
def get_jobs(self):
'''Get the jobs for the user
Returns:
List of dictionaries, containing the job information
'''
jobs = self._job_api.GetJob()
jobs = [sh.swagger_type_to_dict(j) for j in jobs.jobs]
return jobs
def get_task(self, task_id):
'''get a task from the platform task manager
Args:
task_id(string): The id of the task
Returns:
A dictionary with the information about the task
'''
task = self._task_api.GetTaskArg(task_id)
return sh.swagger_type_to_dict(task)
def get_task(self, task_id):
'''get a task from the platform task manager
Args:
task_id(string): The id of the task
Returns:
A dictionary with the information about the task
'''
task = self._task_api.GetTaskArg(task_id)
return sh.swagger_type_to_dict(task)
def register_task(self,
task_filepath,
git_commit,
git_repo,
customizations,
properties,
file_filters=None,
base_env='',
env_vars=None):
'''Register a task with the Platform Task Manager
Args:
task_filepath(string): The path to the file containing the task from the repository root
git_commit(string): The commit used when the task is checked out
git_repo(string): The repo that contains the task code
customizations(dict): where key is 'python', and value is a list of packages
required for the running the task
ex: {'python': ['BeautifulSoup>=3.2.0', 'numpy==1.6.2']}
properties(dict): where key is a string representing a property of the task
(name, caption, state, categories, accepts, returns, compatible_queues,
description, author)
file_filters(dict): Contains keys 'ignored' and 'wanted'
The values for these keys are lists of shell-style
globs of paths to be ignored or wanted
base_env: The base environment, currently not used
env_vars(dict): A dictionary of environment variables. Key is the variable
name and the value is the value
Returns:
task_info(dict): the result of registering the task.
A dictionary with the key 'task_id'
Raises:
'''
args = {}
args['task_filepath'] = task_filepath
args['git_commit'] = git_commit
args['git_repo'] = git_repo
args['requirements'] = {}
args['requirements']['customizations'] = customizations
args['requirements']['file_filters'] = file_filters or {}
args['requirements']['base_env'] = base_env or 'none'
args['requirements']['env_vars'] = env_vars or {}
args['properties'] = JSONable.data_hierachy_pre_json(properties)
body = sh.swagger_create_type(PostTaskSchema.PostTaskSchema, args)
return sh.swagger_type_to_dict(self._task_api.PostTask(body))
def register_task(self, task_filepath,
git_commit, git_repo, customizations, properties,
file_filters=None, base_env='', env_vars=None):
'''Register a task with the Platform Task Manager
Args:
task_filepath(string): The path to the file containing the task from the repository root
git_commit(string): The commit used when the task is checked out
git_repo(string): The repo that contains the task code
customizations(dict): where key is 'python', and value is a list of packages
required for the running the task
ex: {'python': ['BeautifulSoup>=3.2.0', 'numpy==1.6.2']}
properties(dict): where key is a string representing a property of the task
(name, caption, state, categories, accepts, returns, compatible_queues,
description, author)
file_filters(dict): Contains keys 'ignored' and 'wanted'
The values for these keys are lists of shell-style
globs of paths to be ignored or wanted
base_env: The base environment, currently not used
env_vars(dict): A dictionary of environment variables. Key is the variable
name and the value is the value
Returns:
task_info(dict): the result of registering the task.
A dictionary with the key 'task_id'
Raises:
'''
args = {}
args['task_filepath'] = task_filepath
args['git_commit'] = git_commit
args['git_repo'] = git_repo
args['requirements'] = {}
args['requirements']['customizations'] = customizations
args['requirements']['file_filters'] = file_filters or {}
args['requirements']['base_env'] = base_env or 'none'
args['requirements']['env_vars'] = env_vars or {}
args['properties'] = JSONable.data_hierachy_pre_json(properties)
body = sh.swagger_create_type(PostTaskSchema.PostTaskSchema, args)
return sh.swagger_type_to_dict(self._task_api.PostTask(body))
