예제 #1
0
    def _create_local_source(self, file_name, args=None):
        """Creates a new source using a local file.

        This function is only used from Python 3. No async-prepared.

        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        for key, value in create_args.items():
            if value is not None and (isinstance(value, list)
                                      or isinstance(value, dict)):
                create_args[key] = json.dumps(value)
            elif value is not None and isinstance(value, numbers.Number):
                # the multipart encoder only accepts strings and files
                create_args[key] = str(value)

        code = HTTP_INTERNAL_SERVER_ERROR
        resource_id = None
        location = None
        resource = None
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"
            }
        }

        try:

            if isinstance(file_name, basestring):
                name = os.path.basename(file_name)
                file_handler = open(file_name, "rb")
            else:
                name = 'Stdin input'
                file_handler = file_name
        except IOError:
            sys.exit("ERROR: cannot read training set")

        url = self._add_credentials(self.source_url)
        create_args = self._add_project(create_args, True)
        if GAE_ENABLED:
            try:
                req_options = {
                    'url': url,
                    'method': urlfetch.POST,
                    'headers': SEND_JSON,
                    'data': create_args,
                    'files': {
                        name: file_handler
                    },
                    'validate_certificate': self.verify
                }
                response = urlfetch.fetch(**req_options)
            except urlfetch.Error, exception:
                LOGGER.error("HTTP request error: %s", str(exception))
                return maybe_save(resource_id, self.storage, code, location,
                                  resource, error)
예제 #2
0
    def _create_local_source(self, file_name, args=None):
        """Creates a new source using a local file.

        This function is only used from Python 3. No async-prepared.

        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        for key, value in create_args.items():
            if value is not None and (isinstance(value, list) or
                                      isinstance(value, dict)):
                create_args[key] = json.dumps(value)
            elif value is not None and isinstance(value, numbers.Number):
                # the multipart encoder only accepts strings and files
                create_args[key] = str(value)


        code = HTTP_INTERNAL_SERVER_ERROR
        resource_id = None
        location = None
        resource = None
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"}}

        try:

            if isinstance(file_name, basestring):
                name = os.path.basename(file_name)
                file_handler = open(file_name, "rb")
            else:
                name = 'Stdin input'
                file_handler = file_name
        except IOError:
            sys.exit("ERROR: cannot read training set")

        url = self._add_credentials(self.source_url)
        create_args = self._add_project(create_args, True)
        if GAE_ENABLED:
            try:
                req_options = {
                    'url': url,
                    'method': urlfetch.POST,
                    'headers': SEND_JSON,
                    'data': create_args,
                    'files': {name: file_handler},
                    'validate_certificate': self.verify
                }
                response = urlfetch.fetch(**req_options)
            except urlfetch.Error, exception:
                LOGGER.error("HTTP request error: %s",
                             str(exception))
                return maybe_save(resource_id, self.storage, code,
                                  location, resource, error)
예제 #3
0
    def _create_local_source(self, file_name, args=None):
        """Creates a new source using a local file.

        This function is only used from Python 3. No async-prepared.

        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        for key, value in create_args.items():
            if value is not None and (isinstance(value, list)
                                      or isinstance(value, dict)):
                create_args[key] = json.dumps(value)

        code = HTTP_INTERNAL_SERVER_ERROR
        resource_id = None
        location = None
        resource = None
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"
            }
        }

        try:

            if isinstance(file_name, basestring):
                files = {os.path.basename(file_name): open(file_name, "rb")}
            else:
                files = {'stdin': file_name}
        except IOError:
            sys.exit("ERROR: cannot read training set")

        if GAE_ENABLED:
            try:
                req_options = {
                    'url': self.source_url + self.auth,
                    'method': urlfetch.POST,
                    'headers': SEND_JSON,
                    'data': create_args,
                    'files': files,
                    'validate_certificate': self.verify
                }
                response = urlfetch.fetch(**req_options)
            except urlfetch.Error, exception:
                LOGGER.error("HTTP request error: %s", str(exception))
                return maybe_save(resource_id, self.storage, code, location,
                                  resource, error)
예제 #4
0
    def _create_local_source(self, file_name, args=None):
        """Creates a new source using a local file.

        This function is only used from Python 3. No async-prepared.

        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        for key, value in create_args.items():
            if value is not None and (isinstance(value, list) or
                                      isinstance(value, dict)):
                create_args[key] = json.dumps(value)

        code = HTTP_INTERNAL_SERVER_ERROR
        resource_id = None
        location = None
        resource = None
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"}}

        try:

            if isinstance(file_name, basestring):
                files = {os.path.basename(file_name): open(file_name, "rb")}
            else:
                files = {'stdin': file_name}
        except IOError:
            sys.exit("ERROR: cannot read training set")

        if GAE_ENABLED:
            try:
                req_options = {
                    'url': self.source_url + self.auth,
                    'method': urlfetch.POST,
                    'headers': SEND_JSON,
                    'data': create_args,
                    'files': files,
                    'validate_certificate': self.verify
                }
                response = urlfetch.fetch(**req_options)
            except urlfetch.Error, exception:
                LOGGER.error("HTTP request error: %s",
                             str(exception))
                return maybe_save(resource_id, self.storage, code,
                                  location, resource, error)
예제 #5
0
    def _create_local_source(self, file_name, args=None):
        """Creates a new source using a local file.

        This function is now DEPRECATED as "requests" do not stream the file
        content and that limited the size of local files to a small number of
        GBs.

        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        if 'source_parser' in create_args:
            create_args['source_parser'] = json.dumps(
                create_args['source_parser'])

        code = HTTP_INTERNAL_SERVER_ERROR
        resource_id = None
        location = None
        resource = None
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"
            }
        }

        try:
            files = {os.path.basename(file_name): open(file_name, "rb")}
        except IOError:
            raise IOError("ERROR: cannot read training set")

        try:
            response = requests.post(self.source_url + self.auth,
                                     files=files,
                                     data=create_args,
                                     verify=self.verify)

            code = response.status_code
            if code == HTTP_CREATED:
                location = response.headers['location']
                resource = json.loads(response.content, 'utf-8')
                resource_id = resource['resource']
                error = None
            elif code in [
                    HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
                    HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
            ]:
                error = json.loads(response.content, 'utf-8')
            else:
                LOGGER.error("Unexpected error (%s)", code)
                code = HTTP_INTERNAL_SERVER_ERROR

        except ValueError:
            LOGGER.error("Malformed response")
        except requests.ConnectionError, exc:
            LOGGER.error("Connection error: %s", str(exc))
예제 #6
0
    def _process_source(self,
                        resource_id,
                        location,
                        resource,
                        args=None,
                        progress_bar=False,
                        callback=None,
                        out=sys.stdout):
        """Creates a new source.

        """
        code = HTTP_INTERNAL_SERVER_ERROR
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"
            }
        }

        if args is None:
            args = {}
        args = self._add_project(args, True)

        if progress_bar and callback is not None:
            body, headers = multipart_encode(args, cb=callback)
        else:
            body, headers = multipart_encode(args)

        url = self._add_credentials(self.source_url)

        if GAE_ENABLED:
            try:
                response = urlfetch.fetch(url=url,
                                          payload="".join(body),
                                          method=urlfetch.POST,
                                          headers=headers)
                code = response.status_code
                content = response.content
                if code in [HTTP_CREATED]:
                    if 'location' in response.headers:
                        location = response.headers['location']
                    resource = json_load(response.content)
                    resource_id = resource['resource']
                    error = {}
                elif code in [
                        HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED,
                        HTTP_PAYMENT_REQUIRED, HTTP_FORBIDDEN, HTTP_NOT_FOUND,
                        HTTP_TOO_MANY_REQUESTS
                ]:
                    error = json_load(response.content)
                    LOGGER.error(self.error_message(error, method='create'))
                elif code != HTTP_ACCEPTED:
                    LOGGER.error("Unexpected error (%s)", code)
                    code = HTTP_INTERNAL_SERVER_ERROR
            except urlfetch.Error, exception:
                LOGGER.error("Error establishing connection: %s",
                             str(exception))
예제 #7
0
    def _process_source(self, resource_id, location, resource,
                        args=None, progress_bar=False, callback=None,
                        out=sys.stdout):
        """Creates a new source.

        """
        code = HTTP_INTERNAL_SERVER_ERROR
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"}}

        if args is None:
            args = {}
        args = self._add_project(args, True)

        if progress_bar and callback is not None:
            body, headers = multipart_encode(args, cb=callback)
        else:
            body, headers = multipart_encode(args)

        url = self._add_credentials(self.source_url)

        if GAE_ENABLED:
            try:
                response = urlfetch.fetch(url=url,
                                          payload="".join(body),
                                          method=urlfetch.POST,
                                          headers=headers)
                code = response.status_code
                content = response.content
                if code in [HTTP_CREATED]:
                    if 'location' in response.headers:
                        location = response.headers['location']
                    resource = json_load(response.content)
                    resource_id = resource['resource']
                    error = {}
                elif code in [HTTP_BAD_REQUEST,
                              HTTP_UNAUTHORIZED,
                              HTTP_PAYMENT_REQUIRED,
                              HTTP_FORBIDDEN,
                              HTTP_NOT_FOUND,
                              HTTP_TOO_MANY_REQUESTS]:
                    error = json_load(response.content)
                    LOGGER.error(self.error_message(error, method='create'))
                elif code != HTTP_ACCEPTED:
                    LOGGER.error("Unexpected error (%s)", code)
                    code = HTTP_INTERNAL_SERVER_ERROR
            except urlfetch.Error, exception:
                LOGGER.error("Error establishing connection: %s",
                             str(exception))
예제 #8
0
                        response = urllib2.urlopen(request)
                    else:
                        response = urllib2.urlopen(request)
                except AttributeError:
                    response = urllib2.urlopen(request)
                clear_console_line(out=out)
                reset_console_line(out=out)
                code = response.getcode()
                if code == HTTP_CREATED:
                    location = response.headers['location']
                    content = response.read()
                    resource = json_load(content)
                    resource_id = resource['resource']
                    error = {}
            except ValueError:
                LOGGER.error("Malformed response.")
            except urllib2.HTTPError, exception:
                code = exception.code
                if code in [HTTP_BAD_REQUEST,
                            HTTP_UNAUTHORIZED,
                            HTTP_PAYMENT_REQUIRED,
                            HTTP_NOT_FOUND,
                            HTTP_TOO_MANY_REQUESTS]:
                    content = exception.read()
                    error = json_load(content)
                    LOGGER.error(self.error_message(error, method='create'))
                else:
                    LOGGER.error("Unexpected error (%s)", code)
                    code = HTTP_INTERNAL_SERVER_ERROR

            except urllib2.URLError, exception:
예제 #9
0
class SourceHandler(ResourceHandler):
    """This class is used by the BigML class as
       a mixin that provides the REST calls to sources. It should not
       be instantiated independently.

    """
    def __init__(self):
        """Initializes the SourceHandler. This class is intended to be
           used as a mixin on ResourceHandler, that inherits its
           attributes and basic method from BigMLConnection, and must not be
           instantiated independently.

        """
        self.source_url = self.url + SOURCE_PATH

    def _create_remote_source(self, url, args=None):
        """Creates a new source using a URL

        """
        create_args = {}
        if args is not None:
            create_args.update(args)
        create_args.update({"remote": url})
        body = json.dumps(create_args)
        return self._create(self.source_url, body)

    def _create_inline_source(self, src_obj, args=None):
        """Create source from inline data

           The src_obj data should be a list of rows stored as dict or
           list objects.
        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        # some basic validation
        if (not isinstance(src_obj, list)
                or (not all([isinstance(row, dict) for row in src_obj])
                    and not all([isinstance(row, list) for row in src_obj]))):
            raise TypeError(
                'ERROR: inline source must be a list of dicts or a '
                'list of lists')

        create_args.update({"data": json.dumps(src_obj)})
        body = json.dumps(create_args)
        return self._create(self.source_url, body)

    def _create_local_source(self, file_name, args=None):
        """Creates a new source using a local file.

        This function is now DEPRECATED as "requests" do not stream the file
        content and that limited the size of local files to a small number of
        GBs.

        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        if 'source_parser' in create_args:
            create_args['source_parser'] = json.dumps(
                create_args['source_parser'])

        code = HTTP_INTERNAL_SERVER_ERROR
        resource_id = None
        location = None
        resource = None
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"
            }
        }

        try:
            files = {os.path.basename(file_name): open(file_name, "rb")}
        except IOError:
            raise IOError("ERROR: cannot read training set")

        try:
            response = requests.post(self.source_url + self.auth,
                                     files=files,
                                     data=create_args,
                                     verify=self.verify)

            code = response.status_code
            if code == HTTP_CREATED:
                location = response.headers['location']
                resource = json.loads(response.content, 'utf-8')
                resource_id = resource['resource']
                error = None
            elif code in [
                    HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
                    HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
            ]:
                error = json.loads(response.content, 'utf-8')
            else:
                LOGGER.error("Unexpected error (%s)", code)
                code = HTTP_INTERNAL_SERVER_ERROR

        except ValueError:
            LOGGER.error("Malformed response")
        except requests.ConnectionError, exc:
            LOGGER.error("Connection error: %s", str(exc))
        except requests.Timeout:
            LOGGER.error("Request timed out")
예제 #10
0
    def _process_source(self,
                        resource_id,
                        location,
                        resource,
                        args=None,
                        progress_bar=False,
                        callback=None,
                        out=sys.stdout):
        """Creates a new source.

        """
        code = HTTP_INTERNAL_SERVER_ERROR
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"
            }
        }

        if progress_bar and callback is not None:
            body, headers = multipart_encode(args, cb=callback)
        else:
            body, headers = multipart_encode(args)

        request = urllib2.Request(self.source_url + self.auth, body, headers)

        try:
            # try using the new SSL checking in python 2.7.9
            try:
                if not self.verify and PYTHON_2_7_9:
                    context = ssl.create_default_context(
                        ssl.Purpose.CLIENT_AUTH)
                    context.verify_mode = ssl.CERT_NONE
                    https_handler = StreamingHTTPSHandler(context=context)
                    opener = urllib2.build_opener(https_handler)
                    urllib2.install_opener(opener)
                    response = urllib2.urlopen(request)
                else:
                    response = urllib2.urlopen(request)
            except AttributeError:
                response = urllib2.urlopen(request)
            clear_console_line(out=out)
            reset_console_line(out=out)
            code = response.getcode()
            if code == HTTP_CREATED:
                location = response.headers['location']
                content = response.read()
                resource = json.loads(content, 'utf-8')
                resource_id = resource['resource']
                error = {}
        except ValueError:
            LOGGER.error("Malformed response")
        except urllib2.HTTPError, exception:
            code = exception.code
            if code in [
                    HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
                    HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
            ]:
                content = exception.read()
                error = json.loads(content, 'utf-8')
                LOGGER.error(self.error_message(error, method='create'))
            else:
                LOGGER.error("Unexpected error (%s)", code)
                code = HTTP_INTERNAL_SERVER_ERROR
예제 #11
0
                    HTTP_BAD_REQUEST, HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED,
                    HTTP_NOT_FOUND, HTTP_TOO_MANY_REQUESTS
            ]:
                error = json.loads(response.content, 'utf-8')
            else:
                LOGGER.error("Unexpected error (%s)", code)
                code = HTTP_INTERNAL_SERVER_ERROR

        except ValueError:
            LOGGER.error("Malformed response")
        except requests.ConnectionError, exc:
            LOGGER.error("Connection error: %s", str(exc))
        except requests.Timeout:
            LOGGER.error("Request timed out")
        except requests.RequestException:
            LOGGER.error("Ambiguous exception occurred")

        return {
            'code': code,
            'resource': resource_id,
            'location': location,
            'object': resource,
            'error': error
        }

    def _upload_source(self, args, source, out=sys.stdout):
        """Uploads a source asynchronously.

        """
        def update_progress(param, current, total):
            """Updates source's progress.
예제 #12
0
                        response = urllib2.urlopen(request)
                    else:
                        response = urllib2.urlopen(request)
                except AttributeError:
                    response = urllib2.urlopen(request)
                clear_console_line(out=out)
                reset_console_line(out=out)
                code = response.getcode()
                if code == HTTP_CREATED:
                    location = response.headers['location']
                    content = response.read()
                    resource = json.loads(content, 'utf-8')
                    resource_id = resource['resource']
                    error = {}
            except ValueError:
                LOGGER.error("Malformed response.")
            except urllib2.HTTPError, exception:
                code = exception.code
                if code in [HTTP_BAD_REQUEST,
                            HTTP_UNAUTHORIZED,
                            HTTP_PAYMENT_REQUIRED,
                            HTTP_NOT_FOUND,
                            HTTP_TOO_MANY_REQUESTS]:
                    content = exception.read()
                    error = json.loads(content.decode(), 'utf-8')
                    LOGGER.error(self.error_message(error, method='create'))
                else:
                    LOGGER.error("Unexpected error (%s)", code)
                    code = HTTP_INTERNAL_SERVER_ERROR

            except urllib2.URLError, exception: