def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is only used from Python 3. No async-prepared.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in create_args.items():
if value is not None and (isinstance(value, list)
or isinstance(value, dict)):
create_args[key] = json.dumps(value)
elif value is not None and isinstance(value, numbers.Number):
# the multipart encoder only accepts strings and files
create_args[key] = str(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
try:
if isinstance(file_name, basestring):
name = os.path.basename(file_name)
file_handler = open(file_name, "rb")
else:
name = 'Stdin input'
file_handler = file_name
except IOError:
sys.exit("ERROR: cannot read training set")
url = self._add_credentials(self.source_url)
create_args = self._add_project(create_args, True)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': {
name: file_handler
},
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s", str(exception))
return maybe_save(resource_id, self.storage, code, location,
resource, error)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is only used from Python 3. No async-prepared.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in create_args.items():
if value is not None and (isinstance(value, list) or
isinstance(value, dict)):
create_args[key] = json.dumps(value)
elif value is not None and isinstance(value, numbers.Number):
# the multipart encoder only accepts strings and files
create_args[key] = str(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
try:
if isinstance(file_name, basestring):
name = os.path.basename(file_name)
file_handler = open(file_name, "rb")
else:
name = 'Stdin input'
file_handler = file_name
except IOError:
sys.exit("ERROR: cannot read training set")
url = self._add_credentials(self.source_url)
create_args = self._add_project(create_args, True)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': {name: file_handler},
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s",
str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is only used from Python 3. No async-prepared.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in create_args.items():
if value is not None and (isinstance(value, list)
or isinstance(value, dict)):
create_args[key] = json.dumps(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
try:
if isinstance(file_name, basestring):
files = {os.path.basename(file_name): open(file_name, "rb")}
else:
files = {'stdin': file_name}
except IOError:
sys.exit("ERROR: cannot read training set")
if GAE_ENABLED:
try:
req_options = {
'url': self.source_url + self.auth,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': files,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s", str(exception))
return maybe_save(resource_id, self.storage, code, location,
resource, error)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is only used from Python 3. No async-prepared.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in create_args.items():
if value is not None and (isinstance(value, list) or
isinstance(value, dict)):
create_args[key] = json.dumps(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
try:
if isinstance(file_name, basestring):
files = {os.path.basename(file_name): open(file_name, "rb")}
else:
files = {'stdin': file_name}
except IOError:
sys.exit("ERROR: cannot read training set")
if GAE_ENABLED:
try:
req_options = {
'url': self.source_url + self.auth,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': files,
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error, exception:
LOGGER.error("HTTP request error: %s",
str(exception))
return maybe_save(resource_id, self.storage, code,
location, resource, error)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is now DEPRECATED as "requests" do not stream the file
content and that limited the size of local files to a small number of
GBs.
"""
create_args = {}
if args is not None:
create_args.update(args)
if 'source_parser' in create_args:
create_args['source_parser'] = json.dumps(
create_args['source_parser'])
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
try:
files = {os.path.basename(file_name): open(file_name, "rb")}
except IOError:
raise IOError("ERROR: cannot read training set")
try:
response = requests.post(self.source_url + self.auth,
files=files,
data=create_args,
verify=self.verify)
code = response.status_code
if code == HTTP_CREATED:
location = response.headers['location']
resource = json.loads(response.content, 'utf-8')
resource_id = resource['resource']
error = None
elif code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
]:
error = json.loads(response.content, 'utf-8')
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError, exc:
LOGGER.error("Connection error: %s", str(exc))
def _process_source(self,
resource_id,
location,
resource,
args=None,
progress_bar=False,
callback=None,
out=sys.stdout):
"""Creates a new source.
"""
code = HTTP_INTERNAL_SERVER_ERROR
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
if args is None:
args = {}
args = self._add_project(args, True)
if progress_bar and callback is not None:
body, headers = multipart_encode(args, cb=callback)
else:
body, headers = multipart_encode(args)
url = self._add_credentials(self.source_url)
if GAE_ENABLED:
try:
response = urlfetch.fetch(url=url,
payload="".join(body),
method=urlfetch.POST,
headers=headers)
code = response.status_code
content = response.content
if code in [HTTP_CREATED]:
if 'location' in response.headers:
location = response.headers['location']
resource = json_load(response.content)
resource_id = resource['resource']
error = {}
elif code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED, HTTP_FORBIDDEN, HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS
]:
error = json_load(response.content)
LOGGER.error(self.error_message(error, method='create'))
elif code != HTTP_ACCEPTED:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except urlfetch.Error, exception:
LOGGER.error("Error establishing connection: %s",
str(exception))
def _process_source(self, resource_id, location, resource,
args=None, progress_bar=False, callback=None,
out=sys.stdout):
"""Creates a new source.
"""
code = HTTP_INTERNAL_SERVER_ERROR
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
if args is None:
args = {}
args = self._add_project(args, True)
if progress_bar and callback is not None:
body, headers = multipart_encode(args, cb=callback)
else:
body, headers = multipart_encode(args)
url = self._add_credentials(self.source_url)
if GAE_ENABLED:
try:
response = urlfetch.fetch(url=url,
payload="".join(body),
method=urlfetch.POST,
headers=headers)
code = response.status_code
content = response.content
if code in [HTTP_CREATED]:
if 'location' in response.headers:
location = response.headers['location']
resource = json_load(response.content)
resource_id = resource['resource']
error = {}
elif code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_FORBIDDEN,
HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS]:
error = json_load(response.content)
LOGGER.error(self.error_message(error, method='create'))
elif code != HTTP_ACCEPTED:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except urlfetch.Error, exception:
LOGGER.error("Error establishing connection: %s",
str(exception))
response = urllib2.urlopen(request)
else:
response = urllib2.urlopen(request)
except AttributeError:
response = urllib2.urlopen(request)
clear_console_line(out=out)
reset_console_line(out=out)
code = response.getcode()
if code == HTTP_CREATED:
location = response.headers['location']
content = response.read()
resource = json_load(content)
resource_id = resource['resource']
error = {}
except ValueError:
LOGGER.error("Malformed response.")
except urllib2.HTTPError, exception:
code = exception.code
if code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS]:
content = exception.read()
error = json_load(content)
LOGGER.error(self.error_message(error, method='create'))
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except urllib2.URLError, exception:
class SourceHandler(ResourceHandler):
"""This class is used by the BigML class as
a mixin that provides the REST calls to sources. It should not
be instantiated independently.
"""
def __init__(self):
"""Initializes the SourceHandler. This class is intended to be
used as a mixin on ResourceHandler, that inherits its
attributes and basic method from BigMLConnection, and must not be
instantiated independently.
"""
self.source_url = self.url + SOURCE_PATH
def _create_remote_source(self, url, args=None):
"""Creates a new source using a URL
"""
create_args = {}
if args is not None:
create_args.update(args)
create_args.update({"remote": url})
body = json.dumps(create_args)
return self._create(self.source_url, body)
def _create_inline_source(self, src_obj, args=None):
"""Create source from inline data
The src_obj data should be a list of rows stored as dict or
list objects.
"""
create_args = {}
if args is not None:
create_args.update(args)
# some basic validation
if (not isinstance(src_obj, list)
or (not all([isinstance(row, dict) for row in src_obj])
and not all([isinstance(row, list) for row in src_obj]))):
raise TypeError(
'ERROR: inline source must be a list of dicts or a '
'list of lists')
create_args.update({"data": json.dumps(src_obj)})
body = json.dumps(create_args)
return self._create(self.source_url, body)
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
This function is now DEPRECATED as "requests" do not stream the file
content and that limited the size of local files to a small number of
GBs.
"""
create_args = {}
if args is not None:
create_args.update(args)
if 'source_parser' in create_args:
create_args['source_parser'] = json.dumps(
create_args['source_parser'])
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
try:
files = {os.path.basename(file_name): open(file_name, "rb")}
except IOError:
raise IOError("ERROR: cannot read training set")
try:
response = requests.post(self.source_url + self.auth,
files=files,
data=create_args,
verify=self.verify)
code = response.status_code
if code == HTTP_CREATED:
location = response.headers['location']
resource = json.loads(response.content, 'utf-8')
resource_id = resource['resource']
error = None
elif code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
]:
error = json.loads(response.content, 'utf-8')
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError, exc:
LOGGER.error("Connection error: %s", str(exc))
except requests.Timeout:
LOGGER.error("Request timed out")
def _process_source(self,
resource_id,
location,
resource,
args=None,
progress_bar=False,
callback=None,
out=sys.stdout):
"""Creates a new source.
"""
code = HTTP_INTERNAL_SERVER_ERROR
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
if progress_bar and callback is not None:
body, headers = multipart_encode(args, cb=callback)
else:
body, headers = multipart_encode(args)
request = urllib2.Request(self.source_url + self.auth, body, headers)
try:
# try using the new SSL checking in python 2.7.9
try:
if not self.verify and PYTHON_2_7_9:
context = ssl.create_default_context(
ssl.Purpose.CLIENT_AUTH)
context.verify_mode = ssl.CERT_NONE
https_handler = StreamingHTTPSHandler(context=context)
opener = urllib2.build_opener(https_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(request)
else:
response = urllib2.urlopen(request)
except AttributeError:
response = urllib2.urlopen(request)
clear_console_line(out=out)
reset_console_line(out=out)
code = response.getcode()
if code == HTTP_CREATED:
location = response.headers['location']
content = response.read()
resource = json.loads(content, 'utf-8')
resource_id = resource['resource']
error = {}
except ValueError:
LOGGER.error("Malformed response")
except urllib2.HTTPError, exception:
code = exception.code
if code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
]:
content = exception.read()
error = json.loads(content, 'utf-8')
LOGGER.error(self.error_message(error, method='create'))
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
]:
error = json.loads(response.content, 'utf-8')
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
except requests.ConnectionError, exc:
LOGGER.error("Connection error: %s", str(exc))
except requests.Timeout:
LOGGER.error("Request timed out")
except requests.RequestException:
LOGGER.error("Ambiguous exception occurred")
return {
'code': code,
'resource': resource_id,
'location': location,
'object': resource,
'error': error
}
def _upload_source(self, args, source, out=sys.stdout):
"""Uploads a source asynchronously.
"""
def update_progress(param, current, total):
"""Updates source's progress.
response = urllib2.urlopen(request)
else:
response = urllib2.urlopen(request)
except AttributeError:
response = urllib2.urlopen(request)
clear_console_line(out=out)
reset_console_line(out=out)
code = response.getcode()
if code == HTTP_CREATED:
location = response.headers['location']
content = response.read()
resource = json.loads(content, 'utf-8')
resource_id = resource['resource']
error = {}
except ValueError:
LOGGER.error("Malformed response.")
except urllib2.HTTPError, exception:
code = exception.code
if code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS]:
content = exception.read()
error = json.loads(content.decode(), 'utf-8')
LOGGER.error(self.error_message(error, method='create'))
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except urllib2.URLError, exception: