def _process_source(self,
resource_id,
location,
resource,
args=None,
progress_bar=False,
callback=None,
out=sys.stdout):
"""Creates a new source.
"""
code = HTTP_INTERNAL_SERVER_ERROR
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
if args is None:
args = {}
args = self._add_project(args, True)
if progress_bar and callback is not None:
body, headers = multipart_encode(args, cb=callback)
else:
body, headers = multipart_encode(args)
url = self._add_credentials(self.source_url)
if GAE_ENABLED:
try:
response = urlfetch.fetch(url=url,
payload="".join(body),
method=urlfetch.POST,
headers=headers)
code = response.status_code
content = response.content
if code in [HTTP_CREATED]:
if 'location' in response.headers:
location = response.headers['location']
resource = json_load(response.content)
resource_id = resource['resource']
error = {}
elif code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED, HTTP_FORBIDDEN, HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS
]:
error = json_load(response.content)
LOGGER.error(self.error_message(error, method='create'))
elif code != HTTP_ACCEPTED:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except urlfetch.Error, exception:
LOGGER.error("Error establishing connection: %s",
str(exception))
def _process_source(self, resource_id, location, resource,
args=None, progress_bar=False, callback=None,
out=sys.stdout):
"""Creates a new source.
"""
code = HTTP_INTERNAL_SERVER_ERROR
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"}}
if args is None:
args = {}
args = self._add_project(args, True)
if progress_bar and callback is not None:
body, headers = multipart_encode(args, cb=callback)
else:
body, headers = multipart_encode(args)
url = self._add_credentials(self.source_url)
if GAE_ENABLED:
try:
response = urlfetch.fetch(url=url,
payload="".join(body),
method=urlfetch.POST,
headers=headers)
code = response.status_code
content = response.content
if code in [HTTP_CREATED]:
if 'location' in response.headers:
location = response.headers['location']
resource = json_load(response.content)
resource_id = resource['resource']
error = {}
elif code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_FORBIDDEN,
HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS]:
error = json_load(response.content)
LOGGER.error(self.error_message(error, method='create'))
elif code != HTTP_ACCEPTED:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except urlfetch.Error, exception:
LOGGER.error("Error establishing connection: %s",
str(exception))
context.verify_mode = ssl.CERT_NONE
https_handler = StreamingHTTPSHandler(context=context)
opener = urllib2.build_opener(https_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(request)
else:
response = urllib2.urlopen(request)
except AttributeError:
response = urllib2.urlopen(request)
clear_console_line(out=out)
reset_console_line(out=out)
code = response.getcode()
if code == HTTP_CREATED:
location = response.headers['location']
content = response.read()
resource = json_load(content)
resource_id = resource['resource']
error = {}
except ValueError:
LOGGER.error("Malformed response.")
except urllib2.HTTPError, exception:
code = exception.code
if code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS]:
content = exception.read()
error = json_load(content)
LOGGER.error(self.error_message(error, method='create'))
else:
def _create_local_source(self, file_name, args=None):
"""Creates a new source using a local file.
"""
create_args = {}
if args is not None:
create_args.update(args)
for key, value in list(create_args.items()):
if value is not None and isinstance(value, (list, dict)):
create_args[key] = json.dumps(value)
elif value is not None and isinstance(value, numbers.Number):
# the multipart encoder only accepts strings and files
create_args[key] = str(value)
code = HTTP_INTERNAL_SERVER_ERROR
resource_id = None
location = None
resource = None
error = {
"status": {
"code": code,
"message": "The resource couldn't be created"
}
}
try:
if isinstance(file_name, str):
name = os.path.basename(file_name)
file_handler = open(file_name, "rb")
else:
name = 'Stdin input'
file_handler = file_name
except IOError:
sys.exit("ERROR: cannot read training set")
url = self._add_credentials(self.source_url)
create_args = self._add_project(create_args, True)
if GAE_ENABLED:
try:
req_options = {
'url': url,
'method': urlfetch.POST,
'headers': SEND_JSON,
'data': create_args,
'files': {
name: file_handler
},
'validate_certificate': self.verify
}
response = urlfetch.fetch(**req_options)
except urlfetch.Error as exception:
LOGGER.error("HTTP request error: %s", str(exception))
return maybe_save(resource_id, self.storage, code, location,
resource, error)
else:
try:
files = {
"file": (name, file_handler, mimetypes.guess_type(name)[0])
}
files.update(create_args)
multipart = MultipartEncoder(fields=files)
response = requests.post( \
url,
headers={'Content-Type': multipart.content_type},
data=multipart, verify=self.verify)
except (requests.ConnectionError, requests.Timeout,
requests.RequestException) as exc:
LOGGER.error("HTTP request error: %s", str(exc))
code = HTTP_INTERNAL_SERVER_ERROR
return maybe_save(resource_id, self.storage, code, location,
resource, error)
try:
code = response.status_code
if code == HTTP_CREATED:
location = response.headers['location']
resource = json_load(response.content)
resource_id = resource['resource']
error = None
elif code in [
HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
]:
error = json_load(response.content)
else:
LOGGER.error("Unexpected error (%s)", code)
code = HTTP_INTERNAL_SERVER_ERROR
except ValueError:
LOGGER.error("Malformed response")
return maybe_save(resource_id, self.storage, code, location, resource,
error)
context.verify_mode = ssl.CERT_NONE
https_handler = StreamingHTTPSHandler(context=context)
opener = urllib2.build_opener(https_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(request)
else:
response = urllib2.urlopen(request)
except AttributeError:
response = urllib2.urlopen(request)
clear_console_line(out=out)
reset_console_line(out=out)
code = response.getcode()
if code == HTTP_CREATED:
location = response.headers['location']
content = response.read()
resource = json_load(content)
resource_id = resource['resource']
error = {}
except ValueError:
LOGGER.error("Malformed response.")
except urllib2.HTTPError, exception:
code = exception.code
if code in [HTTP_BAD_REQUEST,
HTTP_UNAUTHORIZED,
HTTP_PAYMENT_REQUIRED,
HTTP_NOT_FOUND,
HTTP_TOO_MANY_REQUESTS]:
content = exception.read()
error = json_load(content)
LOGGER.error(self.error_message(error, method='create'))
else: