def get_fields(self, resource): """Retrieve fields used by a resource. Returns a dictionary with the fields that uses the resource keyed by Id. """ def _get_fields_key(resource): """Returns the fields key from a resource dict """ if resource['code'] in [HTTP_OK, HTTP_ACCEPTED]: if (MODEL_RE.match(resource_id) or ANOMALY_RE.match(resource_id)): return resource['object']['model']['model_fields'] elif CLUSTER_RE.match(resource_id): return resource['object']['clusters']['fields'] elif CORRELATION_RE.match(resource_id): return resource['object']['correlations']['fields'] elif STATISTICAL_TEST_RE.match(resource_id): return resource['object']['statistical_tests']['fields'] elif LOGISTIC_REGRESSION_RE.match(resource_id): return resource['object']['logistic_regression']['fields'] elif ASSOCIATION_RE.match(resource_id): return resource['object']['associations']['fields'] elif TOPIC_MODEL_RE.match(resource_id): return resource['object']['topic_model']['fields'] elif TIME_SERIES_RE.match(resource_id): return resource['object']['time_series']['fields'] elif DEEPNET_RE.match(resource_id): return resource['object']['deepnet']['fields'] elif SAMPLE_RE.match(resource_id): return dict([ (field['id'], field) for field in resource['object']['sample']['fields'] ]) else: return resource['object']['fields'] return None if isinstance(resource, dict) and 'resource' in resource: resource_id = resource['resource'] elif (isinstance(resource, basestring) and (SOURCE_RE.match(resource) or DATASET_RE.match(resource) or MODEL_RE.match(resource) or PREDICTION_RE.match(resource))): resource_id = resource resource = self._get("%s%s" % (self.url, resource_id)) else: LOGGER.error("Wrong resource id") return # Tries to extract fields information from resource dict. If it fails, # a get remote call is used to retrieve the resource by id. fields = None try: fields = _get_fields_key(resource) except KeyError: resource = self._get("%s%s" % (self.url, resource_id)) fields = _get_fields_key(resource) return fields
def get_fields(self, resource): """Retrieve fields used by a resource. Returns a dictionary with the fields that uses the resource keyed by Id. """ if isinstance(resource, dict) and 'resource' in resource: resource_id = resource['resource'] elif isinstance(resource, str) and get_resource_type(resource) \ in RESOURCES_WITH_FIELDS: resource_id = resource resource = self.retrieve_resource(resource, query_string=ALL_FIELDS) else: LOGGER.error("Wrong resource id") return # Tries to extract fields information from resource dict. If it fails, # a get remote call is used to retrieve the resource by id. fields = None try: fields = get_fields(resource) except KeyError: resource = self._get("%s%s" % (self.url, resource_id)) fields = get_fields(resource) return fields
def get_fields(self, resource): """Retrieve fields used by a resource. Returns a dictionary with the fields that uses the resource keyed by Id. """ def _get_fields_key(resource): """Returns the fields key from a resource dict """ if resource['code'] in [HTTP_OK, HTTP_ACCEPTED]: if (MODEL_RE.match(resource_id) or ANOMALY_RE.match(resource_id)): return resource['object']['model']['model_fields'] elif CLUSTER_RE.match(resource_id): return resource['object']['clusters']['fields'] elif CORRELATION_RE.match(resource_id): return resource['object']['correlations']['fields'] elif STATISTICAL_TEST_RE.match(resource_id): return resource['object']['statistical_tests']['fields'] elif STATISTICAL_TEST_RE.match(resource_id): return resource['object']['statistical_tests']['fields'] elif LOGISTIC_REGRESSION_RE.match(resource_id): return resource['object']['logistic_regression']['fields'] elif ASSOCIATION_RE.match(resource_id): return resource['object']['associations']['fields'] elif TOPIC_MODEL_RE.match(resource_id): return resource['object']['topic_model']['fields'] elif SAMPLE_RE.match(resource_id): return dict([(field['id'], field) for field in resource['object']['sample']['fields']]) else: return resource['object']['fields'] return None if isinstance(resource, dict) and 'resource' in resource: resource_id = resource['resource'] elif (isinstance(resource, basestring) and ( SOURCE_RE.match(resource) or DATASET_RE.match(resource) or MODEL_RE.match(resource) or PREDICTION_RE.match(resource))): resource_id = resource resource = self._get("%s%s" % (self.url, resource_id)) else: LOGGER.error("Wrong resource id") return # Tries to extract fields information from resource dict. If it fails, # a get remote call is used to retrieve the resource by id. fields = None try: fields = _get_fields_key(resource) except KeyError: resource = self._get("%s%s" % (self.url, resource_id)) fields = _get_fields_key(resource) return fields
def ok(self, resource, query_string='', wait_time=1, max_requests=None, raise_on_error=False, retries=None, error_retries=None, debug=False): """Waits until the resource is finished or faulty, updates it and returns True on success resource: (string|map) Resource ID or structure query_string: (string) Filters used on the resource attributes wait_time: (integer) Time to sleep between get requests max_requests: (integer) Maximum number of get requests raise_on_error: (boolean) Whether to raise errors or log them retries: (integer) Now `max_requests` (deprecated) error_retries: (integer) Retries for transient HTTP errors debug: (boolean) Whether to print traces for every get call """ if http_ok(resource): try: resource.update( check_resource(resource, query_string=query_string, wait_time=wait_time, retries=max_requests, raise_on_error=raise_on_error, api=self, debug=debug)) if resource['error'] and resource['error'].get( \ 'status', {}).get('type') == c.TRANSIENT and \ error_retries is not None and error_retries > 0: return self.ok(resource, query_string, wait_time, max_requests, raise_on_error, error_retries - 1, debug) else: return True except Exception as err: if error_retries is not None and error_retries > 0: return self.ok(resource, query_string, wait_time, max_requests, raise_on_error, error_retries - 1, debug) else: LOGGER.error("The resource info couldn't be retrieved") if raise_on_error: exception_on_error({ "resource": resource["resource"], "error": err }) else: LOGGER.error("The resource couldn't be created: %s", resource['error']) if raise_on_error: exception_on_error(resource)
def status(self, resource): """Maps status code to string. """ resource_id = get_resource_id(resource) if resource_id: resource = self._get("%s%s" % (self.url, resource_id)) status = get_status(resource) code = status['code'] return STATUSES.get(code, "UNKNOWN") status = get_status(resource) if status['code'] != UPLOADING: LOGGER.error("Wrong resource id") return return STATUSES[UPLOADING]
def status(self, resource): """Maps status code to string. """ resource_id = get_resource_id(resource) if resource_id: resource = self._get("%s%s" % (self.url, resource_id)) status = get_status(resource) code = status['code'] return STATUSES.get(code, "UNKNOWN") else: status = get_status(resource) if status['code'] != UPLOADING: LOGGER.error("Wrong resource id") return return STATUSES[UPLOADING]
def ok(self, resource, query_string='', wait_time=1, retries=None, raise_on_error=False): """Waits until the resource is finished or faulty, updates it and returns True on success """ if http_ok(resource): resource.update(check_resource(resource, query_string=query_string, wait_time=wait_time, retries=retries, raise_on_error=raise_on_error, api=self)) return True else: LOGGER.error("The resource couldn't be created: %s", resource['error'])
def get_fields(self, resource): """Retrieve fields used by a resource. Returns a dictionary with the fields that uses the resource keyed by Id. """ def _get_fields_key(resource): """Returns the fields key from a resource dict """ if resource['code'] in [HTTP_OK, HTTP_ACCEPTED]: if MODEL_RE.match(resource_id): return resource['object']['model']['model_fields'] else: return resource['object']['fields'] return None if isinstance(resource, dict) and 'resource' in resource: resource_id = resource['resource'] elif (isinstance(resource, basestring) and ( SOURCE_RE.match(resource) or DATASET_RE.match(resource) or MODEL_RE.match(resource) or PREDICTION_RE.match(resource))): resource_id = resource resource = self._get("%s%s" % (self.url, resource_id)) else: LOGGER.error("Wrong resource id") return # Tries to extract fields information from resource dict. If it fails, # a get remote call is used to retrieve the resource by id. fields = None try: fields = _get_fields_key(resource) except KeyError: resource = self._get("%s%s" % (self.url, resource_id)) fields = _get_fields_key(resource) return fields
def get_fields(self, resource): """Retrieve fields used by a resource. Returns a dictionary with the fields that uses the resource keyed by Id. """ def _get_fields_key(resource): """Returns the fields key from a resource dict """ if resource['code'] in [HTTP_OK, HTTP_ACCEPTED]: if MODEL_RE.match(resource_id): return resource['object']['model']['model_fields'] else: return resource['object']['fields'] return None if isinstance(resource, dict) and 'resource' in resource: resource_id = resource['resource'] elif (isinstance(resource, basestring) and (SOURCE_RE.match(resource) or DATASET_RE.match(resource) or MODEL_RE.match(resource) or PREDICTION_RE.match(resource))): resource_id = resource resource = self._get("%s%s" % (self.url, resource_id)) else: LOGGER.error("Wrong resource id") return # Tries to extract fields information from resource dict. If it fails, # a get remote call is used to retrieve the resource by id. fields = None try: fields = _get_fields_key(resource) except KeyError: resource = self._get("%s%s" % (self.url, resource_id)) fields = _get_fields_key(resource) return fields
def ok(self, resource, query_string='', wait_time=1, max_requests=None, raise_on_error=False, retries=None, error_retries=None, max_elapsed_estimate=float('inf'), debug=False): """Waits until the resource is finished or faulty, updates it and returns True on success resource: (map) Resource structure query_string: (string) Filters used on the resource attributes wait_time: (number) Time to sleep between get requests max_requests: (integer) Maximum number of get requests raise_on_error: (boolean) Whether to raise errors or log them retries: (integer) Now `max_requests` (deprecated) error_retries: (integer) Retries for transient HTTP errors max_elapsed_estimate: (integer) Elapsed number of seconds that we expect the resource to be finished in. This is not a hard limit for the method to end, but an estimation of time to wait. debug: (boolean) Whether to print traces for every get call """ if http_ok(resource): try: resource.update(check_resource( \ resource, query_string=query_string, wait_time=wait_time, retries=max_requests, max_elapsed_estimate=max_elapsed_estimate, raise_on_error=raise_on_error, api=self, debug=debug)) if resource['error'] and resource['error'].get( \ 'status', {}).get('type') == c.TRANSIENT and \ error_retries is not None and error_retries > 0: return self.ok(resource, query_string, wait_time, max_requests, raise_on_error, retries, error_retries - 1, max_elapsed_estimate, debug) return True except Exception as err: if error_retries is not None and error_retries > 0: return self.ok(resource, query_string, wait_time, max_requests, raise_on_error, retries, error_retries - 1, max_elapsed_estimate, debug) LOGGER.error( "The resource info for %s couldn't" " be retrieved", resource["resource"]) if raise_on_error: exception_on_error({ "resource": resource["resource"], "error": err }) else: LOGGER.error("The resource %s couldn't be retrieved: %s", (resource["location"], resource['error'])) if raise_on_error: exception_on_error(resource)