def _process_source(self, resource_id, location, resource, args=None, progress_bar=False, callback=None, out=sys.stdout): """Creates a new source. """ code = HTTP_INTERNAL_SERVER_ERROR error = { "status": { "code": code, "message": "The resource couldn't be created" } } if args is None: args = {} args = self._add_project(args, True) if progress_bar and callback is not None: body, headers = multipart_encode(args, cb=callback) else: body, headers = multipart_encode(args) url = self._add_credentials(self.source_url) if GAE_ENABLED: try: response = urlfetch.fetch(url=url, payload="".join(body), method=urlfetch.POST, headers=headers) code = response.status_code content = response.content if code in [HTTP_CREATED]: if 'location' in response.headers: location = response.headers['location'] resource = json_load(response.content) resource_id = resource['resource'] error = {} elif code in [ HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS ]: error = json_load(response.content) LOGGER.error(self.error_message(error, method='create')) elif code != HTTP_ACCEPTED: LOGGER.error("Unexpected error (%s)", code) code = HTTP_INTERNAL_SERVER_ERROR except urlfetch.Error, exception: LOGGER.error("Error establishing connection: %s", str(exception))
def _process_source(self, resource_id, location, resource, args=None, progress_bar=False, callback=None, out=sys.stdout): """Creates a new source. """ code = HTTP_INTERNAL_SERVER_ERROR error = { "status": { "code": code, "message": "The resource couldn't be created"}} if args is None: args = {} args = self._add_project(args, True) if progress_bar and callback is not None: body, headers = multipart_encode(args, cb=callback) else: body, headers = multipart_encode(args) url = self._add_credentials(self.source_url) if GAE_ENABLED: try: response = urlfetch.fetch(url=url, payload="".join(body), method=urlfetch.POST, headers=headers) code = response.status_code content = response.content if code in [HTTP_CREATED]: if 'location' in response.headers: location = response.headers['location'] resource = json_load(response.content) resource_id = resource['resource'] error = {} elif code in [HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS]: error = json_load(response.content) LOGGER.error(self.error_message(error, method='create')) elif code != HTTP_ACCEPTED: LOGGER.error("Unexpected error (%s)", code) code = HTTP_INTERNAL_SERVER_ERROR except urlfetch.Error, exception: LOGGER.error("Error establishing connection: %s", str(exception))
context.verify_mode = ssl.CERT_NONE https_handler = StreamingHTTPSHandler(context=context) opener = urllib2.build_opener(https_handler) urllib2.install_opener(opener) response = urllib2.urlopen(request) else: response = urllib2.urlopen(request) except AttributeError: response = urllib2.urlopen(request) clear_console_line(out=out) reset_console_line(out=out) code = response.getcode() if code == HTTP_CREATED: location = response.headers['location'] content = response.read() resource = json_load(content) resource_id = resource['resource'] error = {} except ValueError: LOGGER.error("Malformed response.") except urllib2.HTTPError, exception: code = exception.code if code in [HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED, HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS]: content = exception.read() error = json_load(content) LOGGER.error(self.error_message(error, method='create')) else:
def _create_local_source(self, file_name, args=None): """Creates a new source using a local file. """ create_args = {} if args is not None: create_args.update(args) for key, value in list(create_args.items()): if value is not None and isinstance(value, (list, dict)): create_args[key] = json.dumps(value) elif value is not None and isinstance(value, numbers.Number): # the multipart encoder only accepts strings and files create_args[key] = str(value) code = HTTP_INTERNAL_SERVER_ERROR resource_id = None location = None resource = None error = { "status": { "code": code, "message": "The resource couldn't be created" } } try: if isinstance(file_name, str): name = os.path.basename(file_name) file_handler = open(file_name, "rb") else: name = 'Stdin input' file_handler = file_name except IOError: sys.exit("ERROR: cannot read training set") url = self._add_credentials(self.source_url) create_args = self._add_project(create_args, True) if GAE_ENABLED: try: req_options = { 'url': url, 'method': urlfetch.POST, 'headers': SEND_JSON, 'data': create_args, 'files': { name: file_handler }, 'validate_certificate': self.verify } response = urlfetch.fetch(**req_options) except urlfetch.Error as exception: LOGGER.error("HTTP request error: %s", str(exception)) return maybe_save(resource_id, self.storage, code, location, resource, error) else: try: files = { "file": (name, file_handler, mimetypes.guess_type(name)[0]) } files.update(create_args) multipart = MultipartEncoder(fields=files) response = requests.post( \ url, headers={'Content-Type': multipart.content_type}, data=multipart, verify=self.verify) except (requests.ConnectionError, requests.Timeout, requests.RequestException) as exc: LOGGER.error("HTTP request error: %s", str(exc)) code = HTTP_INTERNAL_SERVER_ERROR return maybe_save(resource_id, self.storage, code, location, resource, error) try: code = response.status_code if code == HTTP_CREATED: location = response.headers['location'] resource = json_load(response.content) resource_id = resource['resource'] error = None elif code in [ HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED, HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS ]: error = json_load(response.content) else: LOGGER.error("Unexpected error (%s)", code) code = HTTP_INTERNAL_SERVER_ERROR except ValueError: LOGGER.error("Malformed response") return maybe_save(resource_id, self.storage, code, location, resource, error)
context.verify_mode = ssl.CERT_NONE https_handler = StreamingHTTPSHandler(context=context) opener = urllib2.build_opener(https_handler) urllib2.install_opener(opener) response = urllib2.urlopen(request) else: response = urllib2.urlopen(request) except AttributeError: response = urllib2.urlopen(request) clear_console_line(out=out) reset_console_line(out=out) code = response.getcode() if code == HTTP_CREATED: location = response.headers['location'] content = response.read() resource = json_load(content) resource_id = resource['resource'] error = {} except ValueError: LOGGER.error("Malformed response.") except urllib2.HTTPError, exception: code = exception.code if code in [HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED, HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS]: content = exception.read() error = json_load(content) LOGGER.error(self.error_message(error, method='create')) else: