def get_fields(self, resource):
"""Retrieve fields used by a resource.
Returns a dictionary with the fields that uses
the resource keyed by Id.
"""
def _get_fields_key(resource):
"""Returns the fields key from a resource dict
"""
if resource['code'] in [HTTP_OK, HTTP_ACCEPTED]:
if (MODEL_RE.match(resource_id)
or ANOMALY_RE.match(resource_id)):
return resource['object']['model']['model_fields']
elif CLUSTER_RE.match(resource_id):
return resource['object']['clusters']['fields']
elif CORRELATION_RE.match(resource_id):
return resource['object']['correlations']['fields']
elif STATISTICAL_TEST_RE.match(resource_id):
return resource['object']['statistical_tests']['fields']
elif LOGISTIC_REGRESSION_RE.match(resource_id):
return resource['object']['logistic_regression']['fields']
elif ASSOCIATION_RE.match(resource_id):
return resource['object']['associations']['fields']
elif TOPIC_MODEL_RE.match(resource_id):
return resource['object']['topic_model']['fields']
elif TIME_SERIES_RE.match(resource_id):
return resource['object']['time_series']['fields']
elif DEEPNET_RE.match(resource_id):
return resource['object']['deepnet']['fields']
elif SAMPLE_RE.match(resource_id):
return dict([
(field['id'], field)
for field in resource['object']['sample']['fields']
])
else:
return resource['object']['fields']
return None
if isinstance(resource, dict) and 'resource' in resource:
resource_id = resource['resource']
elif (isinstance(resource, basestring) and
(SOURCE_RE.match(resource) or DATASET_RE.match(resource)
or MODEL_RE.match(resource) or PREDICTION_RE.match(resource))):
resource_id = resource
resource = self._get("%s%s" % (self.url, resource_id))
else:
LOGGER.error("Wrong resource id")
return
# Tries to extract fields information from resource dict. If it fails,
# a get remote call is used to retrieve the resource by id.
fields = None
try:
fields = _get_fields_key(resource)
except KeyError:
resource = self._get("%s%s" % (self.url, resource_id))
fields = _get_fields_key(resource)
return fields
def get_fields(self, resource):
"""Retrieve fields used by a resource.
Returns a dictionary with the fields that uses
the resource keyed by Id.
"""
if isinstance(resource, dict) and 'resource' in resource:
resource_id = resource['resource']
elif isinstance(resource, str) and get_resource_type(resource) \
in RESOURCES_WITH_FIELDS:
resource_id = resource
resource = self.retrieve_resource(resource,
query_string=ALL_FIELDS)
else:
LOGGER.error("Wrong resource id")
return
# Tries to extract fields information from resource dict. If it fails,
# a get remote call is used to retrieve the resource by id.
fields = None
try:
fields = get_fields(resource)
except KeyError:
resource = self._get("%s%s" % (self.url, resource_id))
fields = get_fields(resource)
return fields
def get_fields(self, resource):
"""Retrieve fields used by a resource.
Returns a dictionary with the fields that uses
the resource keyed by Id.
"""
def _get_fields_key(resource):
"""Returns the fields key from a resource dict
"""
if resource['code'] in [HTTP_OK, HTTP_ACCEPTED]:
if (MODEL_RE.match(resource_id) or
ANOMALY_RE.match(resource_id)):
return resource['object']['model']['model_fields']
elif CLUSTER_RE.match(resource_id):
return resource['object']['clusters']['fields']
elif CORRELATION_RE.match(resource_id):
return resource['object']['correlations']['fields']
elif STATISTICAL_TEST_RE.match(resource_id):
return resource['object']['statistical_tests']['fields']
elif STATISTICAL_TEST_RE.match(resource_id):
return resource['object']['statistical_tests']['fields']
elif LOGISTIC_REGRESSION_RE.match(resource_id):
return resource['object']['logistic_regression']['fields']
elif ASSOCIATION_RE.match(resource_id):
return resource['object']['associations']['fields']
elif TOPIC_MODEL_RE.match(resource_id):
return resource['object']['topic_model']['fields']
elif SAMPLE_RE.match(resource_id):
return dict([(field['id'], field) for field in
resource['object']['sample']['fields']])
else:
return resource['object']['fields']
return None
if isinstance(resource, dict) and 'resource' in resource:
resource_id = resource['resource']
elif (isinstance(resource, basestring) and (
SOURCE_RE.match(resource) or DATASET_RE.match(resource) or
MODEL_RE.match(resource) or PREDICTION_RE.match(resource))):
resource_id = resource
resource = self._get("%s%s" % (self.url, resource_id))
else:
LOGGER.error("Wrong resource id")
return
# Tries to extract fields information from resource dict. If it fails,
# a get remote call is used to retrieve the resource by id.
fields = None
try:
fields = _get_fields_key(resource)
except KeyError:
resource = self._get("%s%s" % (self.url, resource_id))
fields = _get_fields_key(resource)
return fields
def ok(self,
resource,
query_string='',
wait_time=1,
max_requests=None,
raise_on_error=False,
retries=None,
error_retries=None,
debug=False):
"""Waits until the resource is finished or faulty, updates it and
returns True on success
resource: (string|map) Resource ID or structure
query_string: (string) Filters used on the resource attributes
wait_time: (integer) Time to sleep between get requests
max_requests: (integer) Maximum number of get requests
raise_on_error: (boolean) Whether to raise errors or log them
retries: (integer) Now `max_requests` (deprecated)
error_retries: (integer) Retries for transient HTTP errors
debug: (boolean) Whether to print traces for every get call
"""
if http_ok(resource):
try:
resource.update(
check_resource(resource,
query_string=query_string,
wait_time=wait_time,
retries=max_requests,
raise_on_error=raise_on_error,
api=self,
debug=debug))
if resource['error'] and resource['error'].get( \
'status', {}).get('type') == c.TRANSIENT and \
error_retries is not None and error_retries > 0:
return self.ok(resource, query_string, wait_time,
max_requests, raise_on_error,
error_retries - 1, debug)
else:
return True
except Exception as err:
if error_retries is not None and error_retries > 0:
return self.ok(resource, query_string, wait_time,
max_requests, raise_on_error,
error_retries - 1, debug)
else:
LOGGER.error("The resource info couldn't be retrieved")
if raise_on_error:
exception_on_error({
"resource": resource["resource"],
"error": err
})
else:
LOGGER.error("The resource couldn't be created: %s",
resource['error'])
if raise_on_error:
exception_on_error(resource)
def status(self, resource):
"""Maps status code to string.
"""
resource_id = get_resource_id(resource)
if resource_id:
resource = self._get("%s%s" % (self.url, resource_id))
status = get_status(resource)
code = status['code']
return STATUSES.get(code, "UNKNOWN")
status = get_status(resource)
if status['code'] != UPLOADING:
LOGGER.error("Wrong resource id")
return
return STATUSES[UPLOADING]
def status(self, resource):
"""Maps status code to string.
"""
resource_id = get_resource_id(resource)
if resource_id:
resource = self._get("%s%s" % (self.url, resource_id))
status = get_status(resource)
code = status['code']
return STATUSES.get(code, "UNKNOWN")
else:
status = get_status(resource)
if status['code'] != UPLOADING:
LOGGER.error("Wrong resource id")
return
return STATUSES[UPLOADING]
def ok(self, resource, query_string='', wait_time=1,
retries=None, raise_on_error=False):
"""Waits until the resource is finished or faulty, updates it and
returns True on success
"""
if http_ok(resource):
resource.update(check_resource(resource,
query_string=query_string,
wait_time=wait_time,
retries=retries,
raise_on_error=raise_on_error,
api=self))
return True
else:
LOGGER.error("The resource couldn't be created: %s",
resource['error'])
def get_fields(self, resource):
"""Retrieve fields used by a resource.
Returns a dictionary with the fields that uses
the resource keyed by Id.
"""
def _get_fields_key(resource):
"""Returns the fields key from a resource dict
"""
if resource['code'] in [HTTP_OK, HTTP_ACCEPTED]:
if MODEL_RE.match(resource_id):
return resource['object']['model']['model_fields']
else:
return resource['object']['fields']
return None
if isinstance(resource, dict) and 'resource' in resource:
resource_id = resource['resource']
elif (isinstance(resource, basestring) and (
SOURCE_RE.match(resource) or DATASET_RE.match(resource) or
MODEL_RE.match(resource) or PREDICTION_RE.match(resource))):
resource_id = resource
resource = self._get("%s%s" % (self.url, resource_id))
else:
LOGGER.error("Wrong resource id")
return
# Tries to extract fields information from resource dict. If it fails,
# a get remote call is used to retrieve the resource by id.
fields = None
try:
fields = _get_fields_key(resource)
except KeyError:
resource = self._get("%s%s" % (self.url, resource_id))
fields = _get_fields_key(resource)
return fields
def get_fields(self, resource):
"""Retrieve fields used by a resource.
Returns a dictionary with the fields that uses
the resource keyed by Id.
"""
def _get_fields_key(resource):
"""Returns the fields key from a resource dict
"""
if resource['code'] in [HTTP_OK, HTTP_ACCEPTED]:
if MODEL_RE.match(resource_id):
return resource['object']['model']['model_fields']
else:
return resource['object']['fields']
return None
if isinstance(resource, dict) and 'resource' in resource:
resource_id = resource['resource']
elif (isinstance(resource, basestring) and
(SOURCE_RE.match(resource) or DATASET_RE.match(resource)
or MODEL_RE.match(resource) or PREDICTION_RE.match(resource))):
resource_id = resource
resource = self._get("%s%s" % (self.url, resource_id))
else:
LOGGER.error("Wrong resource id")
return
# Tries to extract fields information from resource dict. If it fails,
# a get remote call is used to retrieve the resource by id.
fields = None
try:
fields = _get_fields_key(resource)
except KeyError:
resource = self._get("%s%s" % (self.url, resource_id))
fields = _get_fields_key(resource)
return fields
def ok(self,
resource,
query_string='',
wait_time=1,
max_requests=None,
raise_on_error=False,
retries=None,
error_retries=None,
max_elapsed_estimate=float('inf'),
debug=False):
"""Waits until the resource is finished or faulty, updates it and
returns True on success
resource: (map) Resource structure
query_string: (string) Filters used on the resource attributes
wait_time: (number) Time to sleep between get requests
max_requests: (integer) Maximum number of get requests
raise_on_error: (boolean) Whether to raise errors or log them
retries: (integer) Now `max_requests` (deprecated)
error_retries: (integer) Retries for transient HTTP errors
max_elapsed_estimate: (integer) Elapsed number of seconds that we
expect the resource to be finished in.
This is not a hard limit for the method
to end, but an estimation of time to wait.
debug: (boolean) Whether to print traces for every get call
"""
if http_ok(resource):
try:
resource.update(check_resource( \
resource,
query_string=query_string,
wait_time=wait_time,
retries=max_requests,
max_elapsed_estimate=max_elapsed_estimate,
raise_on_error=raise_on_error,
api=self,
debug=debug))
if resource['error'] and resource['error'].get( \
'status', {}).get('type') == c.TRANSIENT and \
error_retries is not None and error_retries > 0:
return self.ok(resource, query_string, wait_time,
max_requests, raise_on_error, retries,
error_retries - 1, max_elapsed_estimate,
debug)
return True
except Exception as err:
if error_retries is not None and error_retries > 0:
return self.ok(resource, query_string, wait_time,
max_requests, raise_on_error, retries,
error_retries - 1, max_elapsed_estimate,
debug)
LOGGER.error(
"The resource info for %s couldn't"
" be retrieved", resource["resource"])
if raise_on_error:
exception_on_error({
"resource": resource["resource"],
"error": err
})
else:
LOGGER.error("The resource %s couldn't be retrieved: %s",
(resource["location"], resource['error']))
if raise_on_error:
exception_on_error(resource)
