def update(self, flushcache=False): """ Update the task object from the REST Api. The flushcache parameter can be used to force the update, otherwise a cached version of the object will be served when accessing properties of the object. Some methods will flush the cache, like :meth:`submit`, :meth:`abort`, :meth:`wait` and :meth:`instant`. Cache behavior is configurable with :attr:`auto_update` and :attr:`update_cache_time`. :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingTaskException: task does not represent a valid one """ if self._uuid is None: return now = time.time() if (now - self._last_cache) < self._update_cache_time and not flushcache: return resp = self._connection._get(get_url('task update', uuid=self._uuid)) if resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp) self._update(resp.json()) self._last_cache = time.time()
def snapshot(self, interval): """Start snapshooting results. If called, this task's results will be periodically updated, instead of only being available at the end. Snapshots will be taken every *interval* second from the time the task is submitted. :param int interval: the interval in seconds at which to take snapshots :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingTaskException: task does not represent a valid one .. note:: To get the temporary results, call :meth:`download_results`. """ if self._uuid is None: self._snapshots = interval return resp = self._connection._post(get_url('task snapshot', uuid=self._uuid), json={"interval": interval}) if resp.status_code == 400: raise ValueError(interval) elif resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp) self._snapshots = True
def update(self, flushcache=False): """ Update the disk object from the REST Api. The flushcache parameter can be used to force the update, otherwise a cached version of the object will be served when accessing properties of the object. :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials """ if self._uuid is None: return now = time.time() if (now - self._last_cache) < self._update_cache_time and not flushcache: return response = self._connection._get(get_url('disk info', name=self._uuid)) if response.status_code == 404: raise MissingDiskException(response.json()['message']) raise_on_error(response) self._update(response.json()) self._last_cache = time.time()
def directory(self, directory=''): """List files in a directory of the disk. Doesn't go through subdirectories. :param str directory: path of the directory to inspect. Must be unix-like. :rtype: List of :class:`FileInfo`. :returns: Files in the given directory on the :class:`Disk`. :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials .. note:: Paths in results are relative to the *directory* argument. """ self.flush() response = self._connection._get( get_url('ls disk', name=self._uuid, path=directory)) if response.status_code == 404: if response.json()['message'] == 'no such disk': raise MissingDiskException(response.json()['message']) raise_on_error(response) return [FileInfo(**f) for f in response.json()]
def tasks(self): """Get the list of tasks stored on this cluster for this user. :rtype: List of :class:`~qarnot.task.Task`. :returns: Tasks stored on the cluster owned by the user. :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details """ response = self._get(get_url('tasks')) raise_on_error(response) return [Task.from_json(self, task) for task in response.json()]
def user_info(self): """Get information of the current user on the cluster. :rtype: :class:`UserInfo` :returns: Requested information. :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details """ resp = self._get(get_url('user')) raise_on_error(resp) ret = resp.json() return UserInfo(ret)
def disks(self): """Get the list of disks on this cluster for this user. :rtype: List of :class:`~qarnot.disk.Disk`. :returns: Disks on the cluster owned by the user. :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details """ response = self._get(get_url('disk folder')) raise_on_error(response) disks = [Disk.from_json(self, data) for data in response.json()] return disks
def update_file_settings(self, remote_path, **kwargs): settings = dict(**kwargs) if len(settings) < 1: return response = self._connection._put(get_url('update file', name=self._uuid, path=remote_path), json=settings) if response.status_code == 404: if response.json()['message'] == "No such disk": raise MissingDiskException(response.json()['message']) raise_on_error(response)
def create(self): """Create the Disk on the REST API. .. note:: This method should not be used unless if the object was created with the constructor. """ data = {"description": self._description, "locked": self._locked} if self._tags is not None: data["tags"] = self._tags response = self._connection._post(get_url('disk folder'), json=data) if response.status_code == 403: raise MaxDiskException(response.json()['message']) else: raise_on_error(response) self._uuid = response.json()['uuid'] self.update()
def delete(self): """Delete the disk represented by this :class:`Disk`. :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials """ response = self._connection._delete( get_url('disk info', name=self._uuid)) if response.status_code == 404: raise MissingDiskException(response.json()['message']) if response.status_code == 403: raise LockedDiskException(response.json()['message']) raise_on_error(response)
def retrieve_profile(self, name): """Get details of a profile from its name. :rtype: :class:`Profile` :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details """ url = get_url('profile details', profile=name) response = self._get(url) raise_on_error(response) if response.status_code == 404: raise QarnotGenericException(response.json()['message']) return Profile(response.json())
def retrieve_task(self, uuid): """Retrieve a :class:`qarnot.task.Task` from its uuid :param str uuid: Desired task uuid :rtype: :class:`~qarnot.task.Task` :returns: Existing task defined by the given uuid :raises qarnot.exceptions.MissingTaskException: task does not exist :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details """ response = self._get(get_url('task update', uuid=uuid)) if response.status_code == 404: raise MissingTaskException(response.json()['message']) raise_on_error(response) return Task.from_json(self, response.json())
def abort(self): """Abort this task if running. :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingTaskException: task does not exist """ self.update(True) resp = self._connection._post(get_url('task abort', uuid=self._uuid)) if resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp) self.update(True)
def update_resources(self): """Update resources for a running task. Be sure to add new resources first. :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingTaskException: task does not exist """ self.update(True) resp = self._connection._patch(get_url('task update', uuid=self._uuid)) if resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp) self.update(True)
def commit(self): """Replicate local changes on the current object instance to the REST API :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials .. note:: When updating disks' properties, auto update will be disabled until commit is called. """ data = self._to_json() resp = self._connection._put(get_url('task update', uuid=self._uuid), json=data) self._auto_update = self._last_auto_update_state if resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp)
def list_files(self): """List files on the whole disk. :rtype: List of :class:`FileInfo`. :returns: List of the files on the disk. :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials """ self.flush() response = self._connection._get(get_url('tree disk', name=self._uuid)) if response.status_code == 404: raise MissingDiskException(response.json()['message']) raise_on_error(response) return [FileInfo(**f) for f in response.json()]
def retrieve_disk(self, uuid): """Retrieve a :class:`~qarnot.disk.Disk` from its uuid :param str uuid: Desired disk uuid :rtype: :class:`~qarnot.disk.Disk` :returns: Existing disk defined by the given uuid :raises ValueError: no such disk :raises qarnot.exceptions.MissingDiskException: disk does not exist :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details """ response = self._get(get_url('disk info', name=uuid)) if response.status_code == 404: raise MissingDiskException(response.json()['message']) raise_on_error(response) return Disk.from_json(self, response.json())
def commit(self): """Replicate local changes on the current object instance to the REST API :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials .. note:: When updating disks' properties, auto update will be disabled until commit is called. """ data = {"description": self._description, "locked": self._locked} if self._tags is not None: data["tags"] = self._tags self._auto_update = self._last_auto_update_state resp = self._connection._put(get_url('disk info', name=self._uuid), json=data) if resp.status_code == 404: raise MissingDiskException(resp.json()['message']) raise_on_error(resp) self.update(True)
def _retrieve(cls, connection, uuid): """Retrieve a submitted task given its uuid. :param qarnot.connection.Connection connection: the cluster to retrieve the task from :param str uuid: the uuid of the task to retrieve :rtype: Task :returns: The retrieved task. :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingTaskException: no such task """ resp = connection._get(get_url('task update', uuid=uuid)) if resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp) return Task.from_json(connection, resp.json())
def _add_file(self, file_, dest, **kwargs): """Add a file on the disk. :param File file_: an opened Python File :param str dest: name of the remote file (defaults to filename) :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials """ try: file_.seek(0) except AttributeError: pass if dest.endswith('/'): dest = os.path.join(dest, os.path.basename(file_.name)) url = get_url('update file', name=self._uuid, path=os.path.dirname(dest)) try: # If requests_toolbelt is installed, we can use its # MultipartEncoder to stream the upload and save memory overuse from requests_toolbelt import MultipartEncoder # noqa m = MultipartEncoder( fields={'filedata': (os.path.basename(dest), file_)}) response = self._connection._post( url, data=m, headers={'Content-Type': m.content_type}) except ImportError: response = self._connection._post( url, files={'filedata': (os.path.basename(dest), file_)}) if response.status_code == 404: raise MissingDiskException(response.json()['message']) raise_on_error(response) # Update file settings if 'executable' not in kwargs: kwargs['executable'] = self._is_executable(file_) self.update_file_settings(dest, **kwargs) self.update(True)
def profiles(self): """Get list of profiles available on the cluster. :rtype: List of :class:`Profile` :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details """ url = get_url('profiles') response = self._get(url) raise_on_error(response) profiles_list = [] for p in response.json(): url = get_url('profile details', profile=p) response2 = self._get(url) if response2.status_code == 404: continue profiles_list.append(Profile(response2.json())) return profiles_list
def instant(self): """Make a snapshot of the current task. :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingTaskException: task does not exist .. note:: To get the temporary results, call :meth:`download_results`. """ if self._uuid is None: return resp = self._connection._post(get_url('task instant', uuid=self._uuid), json=None) if resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp) self.update(True)
def add_link(self, target, linkname): """Create link between files on the disk :param str target: name of the existing file to duplicate :param str linkname: name of the created file .. warning:: File size is counted twice, this method is meant to save upload time, not space. :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials """ data = [{"target": target, "linkName": linkname}] url = get_url('link disk', name=self._uuid) response = self._connection._post(url, json=data) raise_on_error(response) self.update(True)
def fresh_stderr(self): """Get what has been written on the standard error since last time this function was called or since the task has been submitted. :rtype: :class:`str` :returns: The new error messages since last call. :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingTaskException: task does not exist """ if self._uuid is None: return "" resp = self._connection._post(get_url('task stderr', uuid=self._uuid)) if resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp) return resp.text
def delete_file(self, remote, force=False): """Delete a file from the disk. .. note:: You can also use **del disk[file]** :param str remote: the name of the remote file :param bool force: ignore missing files :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises ValueError: no such file (:exc:`KeyError` with disk['file'] syntax) """ dest = remote.name if isinstance(remote, FileInfo) else remote # Ensure 2 threads do not write on the same file pending = self._filethreads.get(dest) if pending is not None: pending.join() # Remove the file from local cache if present if dest in self._filecache: self._filecache[dest].close() del self._filecache[dest] # The file is not present on the disk so just return return response = self._connection._delete( get_url('update file', name=self._uuid, path=dest)) if response.status_code == 404: if response.json()['message'] == "No such disk": raise MissingDiskException(response.json()['message']) if force and response.status_code == 404: pass else: raise_on_error(response) self.update(True)
def _retrieve(cls, connection, disk_uuid): """Retrieve information of a disk on a cluster. :param :class:`qarnot.connection.Connection` connection: the cluster to get the disk from :param str disk_uuid: the UUID of the disk to retrieve :rtype: :class:`qarnot.disk.Disk` :returns: The retrieved disk. :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials """ response = connection._get(get_url('disk info', name=disk_uuid)) if response.status_code == 404: raise MissingDiskException(response.json()['message']) raise_on_error(response) return cls.from_json(connection, response.json())
def move(self, source, dest): """Move a file or a directory inside a disk. Missing destination path directories can be created. Trailing '/' for directories affect behavior. :param str source: name of the source file :param str dest: name of the destination file .. warning:: No clobber on move. If dest exist move will fail. :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials """ data = [{"source": source, "dest": dest}] url = get_url('move disk', name=self._uuid) response = self._connection._post(url, json=data) raise_on_error(response) self.update(True)
def submit(self): """Submit task to the cluster if it is not already submitted. :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.MaxTaskException: Task quota reached :raises qarnot.exceptions.NotEnoughCreditsException: Not enough credits :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingDiskException: resource disk is not a valid disk .. note:: Will ensure all added files are on the resource disk regardless of their uploading mode. .. note:: To get the results, call :meth:`download_results` once the job is done. """ if self._uuid is not None: return self._state for rdisk in self.resources: rdisk.flush() payload = self._to_json() resp = self._connection._post(get_url('tasks'), json=payload) if resp.status_code == 404: raise disk.MissingDiskException(resp.json()['message']) elif resp.status_code == 403: if resp.json()['message'].startswith( 'Maximum number of disks reached'): raise MaxDiskException(resp.json()['message']) else: raise MaxTaskException(resp.json()['message']) elif resp.status_code == 402: raise NotEnoughCreditsException(resp.json()['message']) raise_on_error(resp) self._uuid = resp.json()['uuid'] if not isinstance(self._snapshots, bool): self.snapshot(self._snapshots) self.update(True)
def stderr(self): """Get the standard error of the task since the submission of the task. :rtype: :class:`str` :returns: The standard error. :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises qarnot.exceptions.MissingTaskException: task does not exist .. note:: The buffer is circular, if stderr is too big, prefer calling :meth:`fresh_stderr` regularly. """ if self._uuid is None: return "" resp = self._connection._get(get_url('task stderr', uuid=self._uuid)) if resp.status_code == 404: raise MissingTaskException(resp.json()['message']) raise_on_error(resp) return resp.text
def get_archive(self, extension='zip', local=None): """Get an archive of this disk's content. :param str extension: in {'tar', 'tgz', 'zip'}, format of the archive to get :param str local: name of the file to output to :rtype: :class:`str` :returns: The filename of the retrieved archive. :raises qarnot.exceptions.MissingDiskException: the disk is not on the server :raises qarnot.exceptions.QarnotGenericException: API general error, see message for details :raises qarnot.exceptions.UnauthorizedException: invalid credentials :raises ValueError: invalid extension format """ response = self._connection._get(get_url('get disk', name=self._uuid, ext=extension), stream=True) if response.status_code == 404: raise MissingDiskException(response.json()['message']) elif response.status_code == 400: raise ValueError('invalid file format : {0}', extension) else: raise_on_error(response) local = local or ".".join([self._uuid, extension]) if os.path.isdir(local): local = os.path.join(local, ".".join([self._uuid, extension])) with open(local, 'wb') as f_local: for elt in response.iter_content(): f_local.write(elt) return local