def _create(self, url, body, verify=None, organization=None):
"""Creates a new remote resource.
Posts `body` in JSON to `url` to create a new remote resource.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
# If a prediction server is in use, the first prediction request might
# return a HTTP_ACCEPTED (202) while the model or ensemble is being
# downloaded.
code = HTTP_ACCEPTED
if verify is None:
verify = self.verify
url = self._add_credentials(url, organization=organization)
body = self._add_project(body, not organization)
while code == HTTP_ACCEPTED:
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'payload': body,
'validate_certificate': verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s", str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
else:
try:
response = requests.post(url,
headers=SEND_JSON,
data=body,
verify=verify)
except (requests.ConnectionError, requests.Timeout,
requests.RequestException), exc:
LOGGER.error("HTTP request error: %s", str(exc))
code = HTTP_INTERNAL_SERVER_ERROR
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _create(self, url, body, verify=None, organization=None):
"""Creates a new remote resource.
Posts `body` in JSON to `url` to create a new remote resource.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
# If a prediction server is in use, the first prediction request might
# return a HTTP_ACCEPTED (202) while the model or ensemble is being
# downloaded.
code = HTTP_ACCEPTED
if verify is None:
verify = self.verify
url = self._add_credentials(url, organization=organization)
body = self._add_project(body, not organization)
while code == HTTP_ACCEPTED:
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'payload': body,
'validate_certificate': verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s",
str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
else:
try:
response = requests.post(url,
headers=SEND_JSON,
data=body, verify=verify)
except (requests.ConnectionError,
requests.Timeout,
requests.RequestException), exc:
LOGGER.error("HTTP request error: %s", str(exc))
code = HTTP_INTERNAL_SERVER_ERROR
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is only used from Python 3. No async-prepared.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in create_args.items():
if value is not None and (isinstance(value, list)
or isinstance(value, dict)):
create_args[key] = json.dumps(value)
elif value is not None and isinstance(value, numbers.Number):
# the multipart encoder only accepts strings and files
create_args[key] = str(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
try:
if isinstance(file_name, basestring):
name = os.path.basename(file_name)
file_handler = open(file_name, "rb")
else:
name = 'Stdin input'
file_handler = file_name
except IOError:
sys.exit("ERROR: cannot read training set")
url = self._add_credentials(self.source_url)
create_args = self._add_project(create_args, True)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': {
name: file_handler
},
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s", str(exception))
return maybe_save(resource_id, self.storage, code, location,
resource, error)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is only used from Python 3. No async-prepared.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in create_args.items():
if value is not None and (isinstance(value, list) or
isinstance(value, dict)):
create_args[key] = json.dumps(value)
elif value is not None and isinstance(value, numbers.Number):
# the multipart encoder only accepts strings and files
create_args[key] = str(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
try:
if isinstance(file_name, basestring):
name = os.path.basename(file_name)
file_handler = open(file_name, "rb")
else:
name = 'Stdin input'
file_handler = file_name
except IOError:
sys.exit("ERROR: cannot read training set")
url = self._add_credentials(self.source_url)
create_args = self._add_project(create_args, True)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': {name: file_handler},
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s",
str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _update(self, url, body):
"""Updates a remote resource.
Uses PUT to update a BigML resource. Only the new fields that
are going to be updated need to be included in the `body`.
Returns a resource wrapped in a dictionary:
code: HTTP_ACCEPTED if the update has been OK or an error
code otherwise.
resource: Resource/id
location: Remote location of the resource.
object: The new updated resource
error: Error code if any. None otherwise
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be updated"}}
try:
response = requests.put(url + self.auth,
headers=SEND_JSON,
data=body, verify=VERIFY)
code = response.status_code
if code == HTTP_ACCEPTED:
location = response.headers['location']
resource = json.loads(response.content, 'utf-8')
resource_id = resource['resource']
error = None
elif code in [HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_METHOD_NOT_ALLOWED]:
error = json.loads(response.content, 'utf-8')
LOGGER.error(error_message(error, method='update'))
else:
LOGGER.error("Unexpected error (%s)" % code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError:
LOGGER.error("Connection error")
except requests.Timeout:
LOGGER.error("Request timed out")
except requests.RequestException:
LOGGER.error("Ambiguous exception occurred")
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _update(self, url, body):
"""Updates a remote resource.
Uses PUT to update a BigML resource. Only the new fields that
are going to be updated need to be included in the `body`.
Returns a resource wrapped in a dictionary:
code: HTTP_ACCEPTED if the update has been OK or an error
code otherwise.
resource: Resource/id
location: Remote location of the resource.
object: The new updated resource
error: Error code if any. None otherwise
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be updated"}}
try:
response = requests.put(url + self.auth,
headers=SEND_JSON,
data=body, verify=VERIFY)
code = response.status_code
if code == HTTP_ACCEPTED:
location = response.headers['location']
resource = json.loads(response.content, 'utf-8')
resource_id = resource['resource']
error = None
elif code in [HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_METHOD_NOT_ALLOWED]:
error = json.loads(response.content, 'utf-8')
LOGGER.error(error_message(error, method='update'))
else:
LOGGER.error("Unexpected error (%s)" % code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError:
LOGGER.error("Connection error")
except requests.Timeout:
LOGGER.error("Request timed out")
except requests.RequestException:
LOGGER.error("Ambiguous exception occurred")
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _create(self, url, body):
"""Creates a new remote resource.
Posts `body` in JSON to `url` to create a new remote resource.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
try:
response = requests.post(url + self.auth,
headers=SEND_JSON,
data=body, verify=VERIFY)
code = response.status_code
if code == HTTP_CREATED:
location = response.headers['location']
resource = json.loads(response.content, 'utf-8')
resource_id = resource['resource']
error = None
elif code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_FORBIDDEN,
HTTP_NOT_FOUND]:
error = json.loads(response.content, 'utf-8')
LOGGER.error(error_message(error, method='create'))
else:
LOGGER.error("Unexpected error (%s)" % code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError:
LOGGER.error("Connection error")
except requests.Timeout:
LOGGER.error("Request timed out")
except requests.RequestException:
LOGGER.error("Ambiguous exception occurred")
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _create(self, url, body):
"""Creates a new remote resource.
Posts `body` in JSON to `url` to create a new remote resource.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
try:
response = requests.post(url + self.auth,
headers=SEND_JSON,
data=body, verify=VERIFY)
code = response.status_code
if code == HTTP_CREATED:
location = response.headers['location']
resource = json.loads(response.content, 'utf-8')
resource_id = resource['resource']
error = None
elif code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_FORBIDDEN,
HTTP_NOT_FOUND]:
error = json.loads(response.content, 'utf-8')
LOGGER.error(error_message(error, method='create'))
else:
LOGGER.error("Unexpected error (%s)" % code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError:
LOGGER.error("Connection error")
except requests.Timeout:
LOGGER.error("Request timed out")
except requests.RequestException:
LOGGER.error("Ambiguous exception occurred")
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _get(self, url, query_string=''):
"""Retrieves a remote resource.
Uses HTTP GET to retrieve a BigML `url`.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": HTTP_INTERNAL_SERVER_ERROR,
"message": "The resource couldn't be retrieved"}}
try:
response = requests.get(url + self.auth + query_string,
headers=ACCEPT_JSON,
verify=VERIFY)
code = response.status_code
if code == HTTP_OK:
resource = json.loads(response.content, 'utf-8')
resource_id = resource['resource']
error = None
elif code in [HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_NOT_FOUND]:
error = json.loads(response.content, 'utf-8')
LOGGER.error(error_message(error, method='get'))
else:
LOGGER.error("Unexpected error (%s)" % code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError:
LOGGER.error("Connection error")
except requests.Timeout:
LOGGER.error("Request timed out")
except requests.RequestException:
LOGGER.error("Ambiguous exception occurred")
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _get(self,
url,
query_string='',
shared_username=None,
shared_api_key=None,
organization=None):
"""Retrieves a remote resource.
Uses HTTP GET to retrieve a BigML `url`.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": HTTP_INTERNAL_SERVER_ERROR,
"message": "The resource couldn't be retrieved"
}
}
auth = (self.auth if shared_username is None else
"?username=%s;api_key=%s" % (shared_username, shared_api_key))
kwargs = {"organization": organization}
if shared_username is not None and shared_api_key is not None:
kwargs.update({"shared_auth": auth})
url = self._add_credentials(url, **kwargs) + query_string
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.GET,
'headers': ACCEPT_JSON,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s", str(exception))
error["status"]["type"] = c.TRANSIENT
return maybe_save(resource_id, self.storage, code, location,
resource, error)
def _get(self, url, query_string=''):
"""Retrieves a remote resource.
Uses HTTP GET to retrieve a BigML `url`.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": HTTP_INTERNAL_SERVER_ERROR,
"message": "The resource couldn't be retrieved"}}
try:
response = requests.get(url + self.auth + query_string,
headers=ACCEPT_JSON,
verify=VERIFY)
code = response.status_code
if code == HTTP_OK:
resource = json.loads(response.content, 'utf-8')
resource_id = resource['resource']
error = None
elif code in [HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_NOT_FOUND]:
error = json.loads(response.content, 'utf-8')
LOGGER.error(error_message(error, method='get'))
else:
LOGGER.error("Unexpected error (%s)" % code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError:
LOGGER.error("Connection error")
except requests.Timeout:
LOGGER.error("Request timed out")
except requests.RequestException:
LOGGER.error("Ambiguous exception occurred")
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is only used from Python 3. No async-prepared.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in create_args.items():
if value is not None and (isinstance(value, list)
or isinstance(value, dict)):
create_args[key] = json.dumps(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
try:
if isinstance(file_name, basestring):
files = {os.path.basename(file_name): open(file_name, "rb")}
else:
files = {'stdin': file_name}
except IOError:
sys.exit("ERROR: cannot read training set")
if GAE_ENABLED:
try:
req_options = {
'url': self.source_url + self.auth,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': files,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s", str(exception))
return maybe_save(resource_id, self.storage, code, location,
resource, error)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is only used from Python 3. No async-prepared.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in create_args.items():
if value is not None and (isinstance(value, list) or
isinstance(value, dict)):
create_args[key] = json.dumps(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
try:
if isinstance(file_name, basestring):
files = {os.path.basename(file_name): open(file_name, "rb")}
else:
files = {'stdin': file_name}
except IOError:
sys.exit("ERROR: cannot read training set")
if GAE_ENABLED:
try:
req_options = {
'url': self.source_url + self.auth,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': files,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s",
str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _get(self, url, query_string='',
shared_username=None, shared_api_key=None, organization=None):
"""Retrieves a remote resource.
Uses HTTP GET to retrieve a BigML `url`.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": HTTP_INTERNAL_SERVER_ERROR,
"message": "The resource couldn't be retrieved"}}
auth = (self.auth if shared_username is None
else "?username=%s;api_key=%s" % (
shared_username, shared_api_key))
kwargs = {"organization": organization}
if shared_username is not None and shared_api_key is not None:
kwargs.update({"shared_auth": auth})
url = self._add_credentials(url, **kwargs) + query_string
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.GET,
'headers': ACCEPT_JSON,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s",
str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _update(self, url, body, organization=None):
"""Updates a remote resource.
Uses PUT to update a BigML resource. Only the new fields that
are going to be updated need to be included in the `body`.
Returns a resource wrapped in a dictionary:
code: HTTP_ACCEPTED if the update has been OK or an error
code otherwise.
resource: Resource/id
location: Remote location of the resource.
object: The new updated resource
error: Error code if any. None otherwise
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be updated"
}
}
url = self._add_credentials(url, organization=organization)
body = self._add_project(body, not organization)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.PUT,
'headers': SEND_JSON,
'payload': body,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s", str(exception))
error["status"]["type"] = c.TRANSIENT
return maybe_save(resource_id, self.storage, code, location,
resource, error)
def _update(self, url, body, organization=None):
"""Updates a remote resource.
Uses PUT to update a BigML resource. Only the new fields that
are going to be updated need to be included in the `body`.
Returns a resource wrapped in a dictionary:
code: HTTP_ACCEPTED if the update has been OK or an error
code otherwise.
resource: Resource/id
location: Remote location of the resource.
object: The new updated resource
error: Error code if any. None otherwise
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be updated"}}
url = self._add_credentials(url, organization=organization)
body = self._add_project(body, not organization)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.PUT,
'headers': SEND_JSON,
'payload': body,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s",
str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s",
str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
else:
try:
response = requests.post(self.source_url + self.auth,
files=files,
data=create_args, verify=self.verify)
except (requests.ConnectionError,
requests.Timeout,
requests.RequestException), exc:
LOGGER.error("HTTP request error: %s", str(exc))
code = HTTP_INTERNAL_SERVER_ERROR
return maybe_save(resource_id, self.storage, code,
location, resource, error)
try:
code = response.status_code
if code == HTTP_CREATED:
location = response.headers['location']
resource = json_load(response.content)
resource_id = resource['resource']
error = None
elif code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS]:
error = json_load(response.content)
else:
LOGGER.error("Unexpected error (%s)" % code)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in list(create_args.items()):
if value is not None and isinstance(value, (list, dict)):
create_args[key] = json.dumps(value)
elif value is not None and isinstance(value, numbers.Number):
# the multipart encoder only accepts strings and files
create_args[key] = str(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
try:
if isinstance(file_name, str):
name = os.path.basename(file_name)
file_handler = open(file_name, "rb")
else:
name = 'Stdin input'
file_handler = file_name
except IOError:
sys.exit("ERROR: cannot read training set")
url = self._add_credentials(self.source_url)
create_args = self._add_project(create_args, True)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': {
name: file_handler
},
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error as exception:
LOGGER.error("HTTP request error: %s", str(exception))
return maybe_save(resource_id, self.storage, code, location,
resource, error)
else:
try:
files = {
"file": (name, file_handler, mimetypes.guess_type(name)[0])
}
files.update(create_args)
multipart = MultipartEncoder(fields=files)
response = requests.post( \
url,
headers={'Content-Type': multipart.content_type},
data=multipart, verify=self.verify)
except (requests.ConnectionError, requests.Timeout,
requests.RequestException) as exc:
LOGGER.error("HTTP request error: %s", str(exc))
code = HTTP_INTERNAL_SERVER_ERROR
return maybe_save(resource_id, self.storage, code, location,
resource, error)
try:
code = response.status_code
if code == HTTP_CREATED:
location = response.headers['location']
resource = json_load(response.content)
resource_id = resource['resource']
error = None
elif code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
]:
error = json_load(response.content)
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
return maybe_save(resource_id, self.storage, code, location, resource,
error)
elif code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_FORBIDDEN,
HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS]:
error = json_load(response.content)
LOGGER.error(self.error_message(error, method='create'))
elif code != HTTP_ACCEPTED:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError, exc:
LOGGER.error("Malformed response: %s", str(exc))
code = HTTP_INTERNAL_SERVER_ERROR
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _get(self, url, query_string='',
shared_username=None, shared_api_key=None):
"""Retrieves a remote resource.
Uses HTTP GET to retrieve a BigML `url`.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
def _create(self, url, body, verify=None, organization=None):
"""Creates a new remote resource.
Posts `body` in JSON to `url` to create a new remote resource.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
# If a prediction server is in use, the first prediction request might
# return a HTTP_ACCEPTED (202) while the model or ensemble is being
# downloaded.
code = HTTP_ACCEPTED
if verify is None:
verify = self.verify
url = self._add_credentials(url, organization=organization)
body = self._add_project(body, not organization)
while code == HTTP_ACCEPTED:
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'payload': body,
'validate_certificate': verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error as exception:
LOGGER.error("HTTP request error: %s", str(exception))
error["status"]["type"] = c.TRANSIENT
return maybe_save(resource_id, self.storage, code,
location, resource, error)
else:
try:
response = requests.post(url,
headers=SEND_JSON,
data=body,
verify=verify)
except (requests.ConnectionError, requests.Timeout,
requests.RequestException) as exc:
LOGGER.error("HTTP request error: %s", str(exc))
code = HTTP_INTERNAL_SERVER_ERROR
error["status"]["type"] = c.TRANSIENT
return maybe_save(resource_id, self.storage, code,
location, resource, error)
try:
code = response.status_code
if code in [HTTP_CREATED, HTTP_OK]:
if 'location' in response.headers:
location = response.headers['location']
resource = json_load(response.content)
resource_id = resource.get('resource')
error = None
elif code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED, HTTP_FORBIDDEN, HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS
]:
error = json_load(response.content)
LOGGER.error(self.error_message(error, method='create'))
elif code != HTTP_ACCEPTED:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError as exc:
LOGGER.error("Malformed response: %s", str(exc))
code = HTTP_INTERNAL_SERVER_ERROR
return maybe_save(resource_id, self.storage, code, location, resource,
error)
def _get(self,
url,
query_string='',
shared_username=None,
shared_api_key=None,
organization=None):
"""Retrieves a remote resource.
Uses HTTP GET to retrieve a BigML `url`.
Returns a BigML resource wrapped in a dictionary that includes:
code: HTTP status code
resource: The resource/id
location: Remote location of the resource
object: The resource itself
error: An error code and message
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": HTTP_INTERNAL_SERVER_ERROR,
"message": "The resource couldn't be retrieved"
}
}
auth = (self.auth if shared_username is None else
"?username=%s;api_key=%s" % (shared_username, shared_api_key))
kwargs = {"organization": organization}
if shared_username is not None and shared_api_key is not None:
kwargs.update({"shared_auth": auth})
url = self._add_credentials(url, **kwargs) + query_string
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.GET,
'headers': ACCEPT_JSON,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error as exception:
LOGGER.error("HTTP request error: %s", str(exception))
error["status"]["type"] = c.TRANSIENT
return maybe_save(resource_id, self.storage, code, location,
resource, error)
else:
try:
response = requests.get(url,
headers=ACCEPT_JSON,
verify=self.verify)
except (requests.ConnectionError, requests.Timeout,
requests.RequestException) as exc:
LOGGER.error("HTTP request error: %s", str(exc))
error["status"]["type"] = c.TRANSIENT
return maybe_save(resource_id, self.storage, code, location,
resource, error)
try:
code = response.status_code
if code == HTTP_OK:
resource = json_load(response.content)
resource_id = resource['resource']
error = None
elif code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS
]:
error = json_load(response.content)
LOGGER.error(self.error_message(error, method='get'))
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError as exc:
if "output_format" in query_string:
# output can be an xml file that is returned without storing
return response.content
LOGGER.error("Malformed response: %s", str(exc))
return maybe_save(resource_id, self.storage, code, location, resource,
error)
def _update(self, url, body, organization=None):
"""Updates a remote resource.
Uses PUT to update a BigML resource. Only the new fields that
are going to be updated need to be included in the `body`.
Returns a resource wrapped in a dictionary:
code: HTTP_ACCEPTED if the update has been OK or an error
code otherwise.
resource: Resource/id
location: Remote location of the resource.
object: The new updated resource
error: Error code if any. None otherwise
"""
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = url
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be updated"
}
}
url = self._add_credentials(url, organization=organization)
body = self._add_project(body, not organization)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.PUT,
'headers': SEND_JSON,
'payload': body,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error as exception:
LOGGER.error("HTTP request error: %s", str(exception))
error["status"]["type"] = c.TRANSIENT
return maybe_save(resource_id, self.storage, code, location,
resource, error)
else:
try:
response = requests.put(url,
headers=SEND_JSON,
data=body,
verify=self.verify)
except (requests.ConnectionError, requests.Timeout,
requests.RequestException) as exc:
LOGGER.error("HTTP request error: %s", str(exc))
error["status"]["type"] = c.TRANSIENT
return maybe_save(resource_id, self.storage, code, location,
resource, error)
try:
code = response.status_code
if code == HTTP_ACCEPTED:
resource = json_load(response.content)
resource_id = resource['resource']
error = None
elif code in [
HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
HTTP_METHOD_NOT_ALLOWED, HTTP_TOO_MANY_REQUESTS
]:
error = json_load(response.content)
LOGGER.error(self.error_message(error, method='update'))
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
return maybe_save(resource_id, self.storage, code, location, resource,
error)
