def mkdir(self, path, ignore_error=False): '''mkdir, analagous to os.mkdir''' parent = self._get_parent(path) if self.isfile(parent): raise TypeError('file with this name already exists') name = os.path.basename(path) try: if self.isroot(parent): body = sh.swagger_create_type(ProjectPostJson.ProjectPostJson, {'_name': name}) entity = self._project.create_project(body) elif self.iscontainer(parent): body = sh.swagger_create_type(FolderPostJson.FolderPostJson, { '_name': name, '_parent': parent._uuid, }) entity = self._folder.create_folder(body) except HTTPError: if ignore_error: return None else: raise OSError('directory already exists') self._add_to_cache(parent, (entity, )) return entity._uuid
def mkdir(self, path, ignore_error=False): '''mkdir, analagous to os.mkdir''' parent = self._get_parent(path) if self.isfile(parent): raise TypeError('file with this name already exists') name = os.path.basename(path) try: if self.isroot(parent): body = sh.swagger_create_type(ProjectPostJson.ProjectPostJson, {'_name': name}) entity = self._project.create_project(body) elif self.iscontainer(parent): body = sh.swagger_create_type(FolderPostJson.FolderPostJson, {'_name': name, '_parent': parent._uuid, }) entity = self._folder.create_folder(body) except HTTPError: if ignore_error: return None else: raise OSError('directory already exists') self._add_to_cache(parent, (entity, )) return entity._uuid
def start_job(self, task_id, output_location, arguments, job_name, cpu_cores=1, total_physical_memory=2 * 1024, requested_queue='cscs_viz'): '''Launch a job on through the platfrom task manager Args: task_id(string): The uuid of the task to be launched output_location(string): Location that where this job saves its outputs arguments: The arguments that are passed to the task, they must match the expectations of the parameters cpu_cores(int): number of requested cores total_physical_memory(int): number of megabytes per node requested_queue(string): Where the job is expected to be launched Returns: A dictionary with information to track the the launched job: job_id(string): the unique identifier of the launched job websocket(string): url to a websocket where all logging of the job can be listened to Raises: ''' arguments = JSONable.data_hierachy_pre_json(arguments) body = sh.swagger_create_type(PostJobSchema.PostJobSchema, locals()) return sh.swagger_type_to_dict(self._job_api.PostJob(body))
def start_job(self, task_id, output_location, arguments, job_name, cpu_cores=1, total_physical_memory=2 * 1024, requested_queue='cscs_viz'): '''Launch a job on through the platfrom task manager Args: task_id(string): The uuid of the task to be launched output_location(string): Location that where this job saves its outputs arguments: The arguments that are passed to the task, they must match the expectations of the parameters cpu_cores(int): number of requested cores total_physical_memory(int): number of megabytes per node requested_queue(string): Where the job is expected to be launched Returns: A dictionary with information to track the the launched job: job_id(string): the unique identifier of the launched job websocket(string): url to a websocket where all logging of the job can be listened to Raises: ''' arguments = JSONable.data_hierachy_pre_json(arguments) body = sh.swagger_create_type(PostJobSchema.PostJobSchema, locals()) return sh.swagger_type_to_dict(self._job_api.PostJob(body))
def register_task(self, task_filepath, git_commit, git_repo, customizations, properties, file_filters=None, base_env='', env_vars=None): '''Register a task with the Platform Task Manager Args: task_filepath(string): The path to the file containing the task from the repository root git_commit(string): The commit used when the task is checked out git_repo(string): The repo that contains the task code customizations(dict): where key is 'python', and value is a list of packages required for the running the task ex: {'python': ['BeautifulSoup>=3.2.0', 'numpy==1.6.2']} properties(dict): where key is a string representing a property of the task (name, caption, state, categories, accepts, returns, compatible_queues, description, author) file_filters(dict): Contains keys 'ignored' and 'wanted' The values for these keys are lists of shell-style globs of paths to be ignored or wanted base_env: The base environment, currently not used env_vars(dict): A dictionary of environment variables. Key is the variable name and the value is the value Returns: task_info(dict): the result of registering the task. A dictionary with the key 'task_id' Raises: ''' args = {} args['task_filepath'] = task_filepath args['git_commit'] = git_commit args['git_repo'] = git_repo args['requirements'] = {} args['requirements']['customizations'] = customizations args['requirements']['file_filters'] = file_filters or {} args['requirements']['base_env'] = base_env or 'none' args['requirements']['env_vars'] = env_vars or {} args['properties'] = JSONable.data_hierachy_pre_json(properties) body = sh.swagger_create_type(PostTaskSchema.PostTaskSchema, args) return sh.swagger_type_to_dict(self._task_api.PostTask(body))
def register_task(self, task_filepath, git_commit, git_repo, customizations, properties, file_filters=None, base_env='', env_vars=None): '''Register a task with the Platform Task Manager Args: task_filepath(string): The path to the file containing the task from the repository root git_commit(string): The commit used when the task is checked out git_repo(string): The repo that contains the task code customizations(dict): where key is 'python', and value is a list of packages required for the running the task ex: {'python': ['BeautifulSoup>=3.2.0', 'numpy==1.6.2']} properties(dict): where key is a string representing a property of the task (name, caption, state, categories, accepts, returns, compatible_queues, description, author) file_filters(dict): Contains keys 'ignored' and 'wanted' The values for these keys are lists of shell-style globs of paths to be ignored or wanted base_env: The base environment, currently not used env_vars(dict): A dictionary of environment variables. Key is the variable name and the value is the value Returns: task_info(dict): the result of registering the task. A dictionary with the key 'task_id' Raises: ''' args = {} args['task_filepath'] = task_filepath args['git_commit'] = git_commit args['git_repo'] = git_repo args['requirements'] = {} args['requirements']['customizations'] = customizations args['requirements']['file_filters'] = file_filters or {} args['requirements']['base_env'] = base_env or 'none' args['requirements']['env_vars'] = env_vars or {} args['properties'] = JSONable.data_hierachy_pre_json(properties) body = sh.swagger_create_type(PostTaskSchema.PostTaskSchema, args) return sh.swagger_type_to_dict(self._task_api.PostTask(body))
def _create_placeholder(self, dst, mimetype, st_attr, extra_attr=None): '''creates a placeholder''' #TODO: Do we allow overwriting of files? #if self._get_entity_by_path(dst) is not None: # raise OSError('Cannot overwrite a directory that already exists') parent = self._get_parent(dst) dst_name = os.path.basename(dst) # create placeholder props = st_attr if st_attr else {} props.update({'_name': dst_name, '_parent': parent._uuid, }) if extra_attr: props.update(extra_attr) if mimetype is not None: props['_contentType'] = str(mimetype) body = sh.swagger_create_type(FilePostJson.FilePostJson, props) entity = self._file.create_file(body) return parent, entity
def _create_placeholder(self, dst, mimetype, st_attr, extra_attr=None): '''creates a placeholder''' #TODO: Do we allow overwriting of files? #if self._get_entity_by_path(dst) is not None: # raise OSError('Cannot overwrite a directory that already exists') parent = self._get_parent(dst) dst_name = os.path.basename(dst) # create placeholder props = st_attr if st_attr else {} props.update({ '_name': dst_name, '_parent': parent._uuid, }) if extra_attr: props.update(extra_attr) if mimetype is not None: props['_contentType'] = str(mimetype) body = sh.swagger_create_type(FilePostJson.FilePostJson, props) entity = self._file.create_file(body) return parent, entity