def update(self, flushcache=False):
"""
Update the task object from the REST Api.
The flushcache parameter can be used to force the update, otherwise a cached version of the object
will be served when accessing properties of the object.
Some methods will flush the cache, like :meth:`submit`, :meth:`abort`, :meth:`wait` and :meth:`instant`.
Cache behavior is configurable with :attr:`auto_update` and :attr:`update_cache_time`.
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingTaskException: task does not represent a
valid one
"""
if self._uuid is None:
return
now = time.time()
if (now -
self._last_cache) < self._update_cache_time and not flushcache:
return
resp = self._connection._get(get_url('task update', uuid=self._uuid))
if resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
self._update(resp.json())
self._last_cache = time.time()
def snapshot(self, interval):
"""Start snapshooting results.
If called, this task's results will be periodically
updated, instead of only being available at the end.
Snapshots will be taken every *interval* second from the time
the task is submitted.
:param int interval: the interval in seconds at which to take snapshots
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingTaskException: task does not represent a
valid one
.. note:: To get the temporary results, call :meth:`download_results`.
"""
if self._uuid is None:
self._snapshots = interval
return
resp = self._connection._post(get_url('task snapshot',
uuid=self._uuid),
json={"interval": interval})
if resp.status_code == 400:
raise ValueError(interval)
elif resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
self._snapshots = True
def update(self, flushcache=False):
"""
Update the disk object from the REST Api.
The flushcache parameter can be used to force the update, otherwise a cached version of the object
will be served when accessing properties of the object.
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
"""
if self._uuid is None:
return
now = time.time()
if (now -
self._last_cache) < self._update_cache_time and not flushcache:
return
response = self._connection._get(get_url('disk info', name=self._uuid))
if response.status_code == 404:
raise MissingDiskException(response.json()['message'])
raise_on_error(response)
self._update(response.json())
self._last_cache = time.time()
def directory(self, directory=''):
"""List files in a directory of the disk. Doesn't go through
subdirectories.
:param str directory: path of the directory to inspect.
Must be unix-like.
:rtype: List of :class:`FileInfo`.
:returns: Files in the given directory on the :class:`Disk`.
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
.. note::
Paths in results are relative to the *directory* argument.
"""
self.flush()
response = self._connection._get(
get_url('ls disk', name=self._uuid, path=directory))
if response.status_code == 404:
if response.json()['message'] == 'no such disk':
raise MissingDiskException(response.json()['message'])
raise_on_error(response)
return [FileInfo(**f) for f in response.json()]
def tasks(self):
"""Get the list of tasks stored on this cluster for this user.
:rtype: List of :class:`~qarnot.task.Task`.
:returns: Tasks stored on the cluster owned by the user.
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
"""
response = self._get(get_url('tasks'))
raise_on_error(response)
return [Task.from_json(self, task) for task in response.json()]
def user_info(self):
"""Get information of the current user on the cluster.
:rtype: :class:`UserInfo`
:returns: Requested information.
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
"""
resp = self._get(get_url('user'))
raise_on_error(resp)
ret = resp.json()
return UserInfo(ret)
def disks(self):
"""Get the list of disks on this cluster for this user.
:rtype: List of :class:`~qarnot.disk.Disk`.
:returns: Disks on the cluster owned by the user.
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
"""
response = self._get(get_url('disk folder'))
raise_on_error(response)
disks = [Disk.from_json(self, data) for data in response.json()]
return disks
def update_file_settings(self, remote_path, **kwargs):
settings = dict(**kwargs)
if len(settings) < 1:
return
response = self._connection._put(get_url('update file',
name=self._uuid,
path=remote_path),
json=settings)
if response.status_code == 404:
if response.json()['message'] == "No such disk":
raise MissingDiskException(response.json()['message'])
raise_on_error(response)
def create(self):
"""Create the Disk on the REST API.
.. note:: This method should not be used unless if the object was created with the constructor.
"""
data = {"description": self._description, "locked": self._locked}
if self._tags is not None:
data["tags"] = self._tags
response = self._connection._post(get_url('disk folder'), json=data)
if response.status_code == 403:
raise MaxDiskException(response.json()['message'])
else:
raise_on_error(response)
self._uuid = response.json()['uuid']
self.update()
def delete(self):
"""Delete the disk represented by this :class:`Disk`.
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
"""
response = self._connection._delete(
get_url('disk info', name=self._uuid))
if response.status_code == 404:
raise MissingDiskException(response.json()['message'])
if response.status_code == 403:
raise LockedDiskException(response.json()['message'])
raise_on_error(response)
def retrieve_profile(self, name):
"""Get details of a profile from its name.
:rtype: :class:`Profile`
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
"""
url = get_url('profile details', profile=name)
response = self._get(url)
raise_on_error(response)
if response.status_code == 404:
raise QarnotGenericException(response.json()['message'])
return Profile(response.json())
def retrieve_task(self, uuid):
"""Retrieve a :class:`qarnot.task.Task` from its uuid
:param str uuid: Desired task uuid
:rtype: :class:`~qarnot.task.Task`
:returns: Existing task defined by the given uuid
:raises qarnot.exceptions.MissingTaskException: task does not exist
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
"""
response = self._get(get_url('task update', uuid=uuid))
if response.status_code == 404:
raise MissingTaskException(response.json()['message'])
raise_on_error(response)
return Task.from_json(self, response.json())
def abort(self):
"""Abort this task if running.
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingTaskException: task does not exist
"""
self.update(True)
resp = self._connection._post(get_url('task abort', uuid=self._uuid))
if resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
self.update(True)
def update_resources(self):
"""Update resources for a running task. Be sure to add new resources first.
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingTaskException: task does not exist
"""
self.update(True)
resp = self._connection._patch(get_url('task update', uuid=self._uuid))
if resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
self.update(True)
def commit(self):
"""Replicate local changes on the current object instance to the REST API
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
.. note:: When updating disks' properties, auto update will be disabled until commit is called.
"""
data = self._to_json()
resp = self._connection._put(get_url('task update', uuid=self._uuid),
json=data)
self._auto_update = self._last_auto_update_state
if resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
def list_files(self):
"""List files on the whole disk.
:rtype: List of :class:`FileInfo`.
:returns: List of the files on the disk.
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
"""
self.flush()
response = self._connection._get(get_url('tree disk', name=self._uuid))
if response.status_code == 404:
raise MissingDiskException(response.json()['message'])
raise_on_error(response)
return [FileInfo(**f) for f in response.json()]
def retrieve_disk(self, uuid):
"""Retrieve a :class:`~qarnot.disk.Disk` from its uuid
:param str uuid: Desired disk uuid
:rtype: :class:`~qarnot.disk.Disk`
:returns: Existing disk defined by the given uuid
:raises ValueError: no such disk
:raises qarnot.exceptions.MissingDiskException: disk does not exist
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
"""
response = self._get(get_url('disk info', name=uuid))
if response.status_code == 404:
raise MissingDiskException(response.json()['message'])
raise_on_error(response)
return Disk.from_json(self, response.json())
def commit(self):
"""Replicate local changes on the current object instance to the REST API
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
.. note:: When updating disks' properties, auto update will be disabled until commit is called.
"""
data = {"description": self._description, "locked": self._locked}
if self._tags is not None:
data["tags"] = self._tags
self._auto_update = self._last_auto_update_state
resp = self._connection._put(get_url('disk info', name=self._uuid),
json=data)
if resp.status_code == 404:
raise MissingDiskException(resp.json()['message'])
raise_on_error(resp)
self.update(True)
def _retrieve(cls, connection, uuid):
"""Retrieve a submitted task given its uuid.
:param qarnot.connection.Connection connection:
the cluster to retrieve the task from
:param str uuid: the uuid of the task to retrieve
:rtype: Task
:returns: The retrieved task.
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingTaskException: no such task
"""
resp = connection._get(get_url('task update', uuid=uuid))
if resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
return Task.from_json(connection, resp.json())
def _add_file(self, file_, dest, **kwargs):
"""Add a file on the disk.
:param File file_: an opened Python File
:param str dest: name of the remote file (defaults to filename)
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
"""
try:
file_.seek(0)
except AttributeError:
pass
if dest.endswith('/'):
dest = os.path.join(dest, os.path.basename(file_.name))
url = get_url('update file',
name=self._uuid,
path=os.path.dirname(dest))
try:
# If requests_toolbelt is installed, we can use its
# MultipartEncoder to stream the upload and save memory overuse
from requests_toolbelt import MultipartEncoder # noqa
m = MultipartEncoder(
fields={'filedata': (os.path.basename(dest), file_)})
response = self._connection._post(
url, data=m, headers={'Content-Type': m.content_type})
except ImportError:
response = self._connection._post(
url, files={'filedata': (os.path.basename(dest), file_)})
if response.status_code == 404:
raise MissingDiskException(response.json()['message'])
raise_on_error(response)
# Update file settings
if 'executable' not in kwargs:
kwargs['executable'] = self._is_executable(file_)
self.update_file_settings(dest, **kwargs)
self.update(True)
def profiles(self):
"""Get list of profiles available on the cluster.
:rtype: List of :class:`Profile`
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
"""
url = get_url('profiles')
response = self._get(url)
raise_on_error(response)
profiles_list = []
for p in response.json():
url = get_url('profile details', profile=p)
response2 = self._get(url)
if response2.status_code == 404:
continue
profiles_list.append(Profile(response2.json()))
return profiles_list
def instant(self):
"""Make a snapshot of the current task.
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingTaskException: task does not exist
.. note:: To get the temporary results, call :meth:`download_results`.
"""
if self._uuid is None:
return
resp = self._connection._post(get_url('task instant', uuid=self._uuid),
json=None)
if resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
self.update(True)
def add_link(self, target, linkname):
"""Create link between files on the disk
:param str target: name of the existing file to duplicate
:param str linkname: name of the created file
.. warning::
File size is counted twice, this method is meant to save upload
time, not space.
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
"""
data = [{"target": target, "linkName": linkname}]
url = get_url('link disk', name=self._uuid)
response = self._connection._post(url, json=data)
raise_on_error(response)
self.update(True)
def fresh_stderr(self):
"""Get what has been written on the standard error since last time
this function was called or since the task has been submitted.
:rtype: :class:`str`
:returns: The new error messages since last call.
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingTaskException: task does not exist
"""
if self._uuid is None:
return ""
resp = self._connection._post(get_url('task stderr', uuid=self._uuid))
if resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
return resp.text
def delete_file(self, remote, force=False):
"""Delete a file from the disk.
.. note::
You can also use **del disk[file]**
:param str remote: the name of the remote file
:param bool force: ignore missing files
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises ValueError: no such file
(:exc:`KeyError` with disk['file'] syntax)
"""
dest = remote.name if isinstance(remote, FileInfo) else remote
# Ensure 2 threads do not write on the same file
pending = self._filethreads.get(dest)
if pending is not None:
pending.join()
# Remove the file from local cache if present
if dest in self._filecache:
self._filecache[dest].close()
del self._filecache[dest]
# The file is not present on the disk so just return
return
response = self._connection._delete(
get_url('update file', name=self._uuid, path=dest))
if response.status_code == 404:
if response.json()['message'] == "No such disk":
raise MissingDiskException(response.json()['message'])
if force and response.status_code == 404:
pass
else:
raise_on_error(response)
self.update(True)
def _retrieve(cls, connection, disk_uuid):
"""Retrieve information of a disk on a cluster.
:param :class:`qarnot.connection.Connection` connection: the cluster
to get the disk from
:param str disk_uuid: the UUID of the disk to retrieve
:rtype: :class:`qarnot.disk.Disk`
:returns: The retrieved disk.
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
"""
response = connection._get(get_url('disk info', name=disk_uuid))
if response.status_code == 404:
raise MissingDiskException(response.json()['message'])
raise_on_error(response)
return cls.from_json(connection, response.json())
def move(self, source, dest):
"""Move a file or a directory inside a disk.
Missing destination path directories can be created.
Trailing '/' for directories affect behavior.
:param str source: name of the source file
:param str dest: name of the destination file
.. warning::
No clobber on move. If dest exist move will fail.
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
"""
data = [{"source": source, "dest": dest}]
url = get_url('move disk', name=self._uuid)
response = self._connection._post(url, json=data)
raise_on_error(response)
self.update(True)
def submit(self):
"""Submit task to the cluster if it is not already submitted.
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.MaxTaskException: Task quota reached
:raises qarnot.exceptions.NotEnoughCreditsException: Not enough credits
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingDiskException:
resource disk is not a valid disk
.. note:: Will ensure all added files are on the resource disk
regardless of their uploading mode.
.. note:: To get the results, call :meth:`download_results` once the job is done.
"""
if self._uuid is not None:
return self._state
for rdisk in self.resources:
rdisk.flush()
payload = self._to_json()
resp = self._connection._post(get_url('tasks'), json=payload)
if resp.status_code == 404:
raise disk.MissingDiskException(resp.json()['message'])
elif resp.status_code == 403:
if resp.json()['message'].startswith(
'Maximum number of disks reached'):
raise MaxDiskException(resp.json()['message'])
else:
raise MaxTaskException(resp.json()['message'])
elif resp.status_code == 402:
raise NotEnoughCreditsException(resp.json()['message'])
raise_on_error(resp)
self._uuid = resp.json()['uuid']
if not isinstance(self._snapshots, bool):
self.snapshot(self._snapshots)
self.update(True)
def stderr(self):
"""Get the standard error of the task
since the submission of the task.
:rtype: :class:`str`
:returns: The standard error.
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises qarnot.exceptions.MissingTaskException: task does not exist
.. note:: The buffer is circular, if stderr is too big, prefer calling
:meth:`fresh_stderr` regularly.
"""
if self._uuid is None:
return ""
resp = self._connection._get(get_url('task stderr', uuid=self._uuid))
if resp.status_code == 404:
raise MissingTaskException(resp.json()['message'])
raise_on_error(resp)
return resp.text
def get_archive(self, extension='zip', local=None):
"""Get an archive of this disk's content.
:param str extension: in {'tar', 'tgz', 'zip'},
format of the archive to get
:param str local: name of the file to output to
:rtype: :class:`str`
:returns:
The filename of the retrieved archive.
:raises qarnot.exceptions.MissingDiskException: the disk is not on the server
:raises qarnot.exceptions.QarnotGenericException: API general error, see message for details
:raises qarnot.exceptions.UnauthorizedException: invalid credentials
:raises ValueError: invalid extension format
"""
response = self._connection._get(get_url('get disk',
name=self._uuid,
ext=extension),
stream=True)
if response.status_code == 404:
raise MissingDiskException(response.json()['message'])
elif response.status_code == 400:
raise ValueError('invalid file format : {0}', extension)
else:
raise_on_error(response)
local = local or ".".join([self._uuid, extension])
if os.path.isdir(local):
local = os.path.join(local, ".".join([self._uuid, extension]))
with open(local, 'wb') as f_local:
for elt in response.iter_content():
f_local.write(elt)
return local
