def delete_source(self, source):
"""Deletes a remote source permanently.
"""
check_resource_type(source,
SOURCE_PATH,
message="A source id is needed.")
source_id = get_source_id(source)
if source_id:
return self._delete("%s%s" % (self.url, source_id))
def update_source(self, source, changes):
"""Updates a source.
Updates remote `source` with `changes'.
"""
check_resource_type(source,
SOURCE_PATH,
message="A source id is needed.")
source_id = get_source_id(source)
if source_id:
body = json.dumps(changes)
return self._update("%s%s" % (self.url, source_id), body)
def get_source(self, source, query_string=''):
"""Retrieves a remote source.
The source parameter should be a string containing the
source id or the dict returned by create_source.
As source is an evolving object that is processed
until it reaches the FINISHED or FAULTY state, thet function will
return a dict that encloses the source values and state info
available at the time it is called.
"""
check_resource_type(source,
SOURCE_PATH,
message="A source id is needed.")
source_id = get_source_id(source)
if source_id:
return self._get("%s%s" % (self.url, source_id),
query_string=query_string)
def create_dataset(self,
origin_resource,
args=None,
wait_time=3,
retries=10):
"""Creates a remote dataset.
Uses a remote resource to create a new dataset using the
arguments in `args`.
The allowed remote resources can be:
- source
- dataset
- list of datasets
- cluster
In the case of using cluster id as origin_resources, a centroid must
also be provided in the args argument. The first centroid is used
otherwise.
If `wait_time` is higher than 0 then the dataset creation
request is not sent until the `source` has been created successfuly.
"""
create_args = {}
if args is not None:
create_args.update(args)
if isinstance(origin_resource, list):
# mutidatasets
create_args = self._set_create_from_datasets_args(
origin_resource,
args=create_args,
wait_time=wait_time,
retries=retries,
key="origin_datasets")
else:
# dataset from source
resource_type = get_resource_type(origin_resource)
if resource_type == SOURCE_PATH:
source_id = get_source_id(origin_resource)
if source_id:
check_resource(source_id,
query_string=TINY_RESOURCE,
wait_time=wait_time,
retries=retries,
raise_on_error=True,
api=self)
create_args.update({"source": source_id})
# dataset from dataset
elif resource_type == DATASET_PATH:
create_args = self._set_create_from_datasets_args(
origin_resource,
args=create_args,
wait_time=wait_time,
retries=retries,
key="origin_dataset")
# dataset from cluster and centroid
elif resource_type == CLUSTER_PATH:
cluster_id = get_cluster_id(origin_resource)
cluster = check_resource(cluster_id,
query_string=TINY_RESOURCE,
wait_time=wait_time,
retries=retries,
raise_on_error=True,
api=self)
if 'centroid' not in create_args:
try:
centroid = list(cluster['object']
['cluster_datasets_ids'].keys())[0]
create_args.update({'centroid': centroid})
except KeyError:
raise KeyError("Failed to generate the dataset. A "
"centroid id is needed in the args "
"argument to generate a dataset from "
"a cluster.")
create_args.update({'cluster': cluster_id})
else:
raise Exception("A source, dataset, list of dataset ids"
" or cluster id plus centroid id are needed"
" to create a"
" dataset. %s found." % resource_type)
body = json.dumps(create_args)
return self._create(self.dataset_url, body)