Example #1
0
    def load(self, dataset, id):
        """
        Load the dataset into the database. If a url parameter 'sample'
        is provided then its value is converted into a boolean. If the value
        equals true we only perform a sample run, else we do a full load.
        """

        # Get our source (and dataset)
        self._get_source(dataset, id)

        # We require that the user can update the dataset
        require.dataset.update(c.dataset)

        # If the source is already running we flash an error declaring that
        # we're already running this source
        if c.source.is_running:
            h.flash_error(_("Already running!"))
        # If the source isn't already running we try to load it (or sample it)
        else:
            try:
                sample = asbool(request.params.get('sample', 'false'))
                load_source.delay(c.source.id, sample)
                # Let the user know we're loading the source
                h.flash_success(_("Now loading..."))
            except Exception as e:
                abort(400, e)

        # Send the user to the editor index page for this dataset
        redirect(h.url_for(controller='editor', action='index',
                           dataset=c.dataset.name))
Example #2
0
    def load(self, dataset, id):
        """
        Load the dataset into the database. If a url parameter 'sample'
        is provided then its value is converted into a boolean. If the value
        equals true we only perform a sample run, else we do a full load.
        """

        # Get our source (and dataset)
        self._get_source(dataset, id)

        # We require that the user can update the dataset
        require.dataset.update(c.dataset)

        # If the source is already running we flash an error declaring that
        # we're already running this source
        if c.source.is_running:
            h.flash_error(_("Already running!"))
        # If the source isn't already running we try to load it (or sample it)
        else:
            try:
                sample = asbool(request.params.get('sample', 'false'))
                load_source.delay(c.source.id, sample)
                # Let the user know we're loading the source
                h.flash_success(_("Now loading..."))
            except Exception as e:
                abort(400, e)

        # Send the user to the editor index page for this dataset
        redirect(
            h.url_for(controller='editor',
                      action='index',
                      dataset=c.dataset.name))
Example #3
0
    def create(self):
        """
        Adds a new dataset dynamically through a POST request
        """

        # User must be authenticated so we should have a user object in
        # c.account, if not abort with error message
        if not c.account:
            abort(status_code=400, detail='user not authenticated')

        # Check if the params are there ('metadata', 'csv_file')
        if len(request.params) != 2:
            abort(status_code=400, detail='incorrect number of params')

        metadata = request.params['metadata'] \
            if 'metadata' in request.params \
            else abort(status_code=400, detail='metadata is missing')

        csv_file = request.params['csv_file'] \
            if 'csv_file' in request.params \
            else abort(status_code=400, detail='csv_file is missing')

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(metadata))
        except:
            abort(status_code=400, detail='JSON model could not be parsed')
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            abort(status_code=400, detail='Model is not well formed')
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = True  # Default value
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account, url=csv_file)

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == csv_file:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))
Example #4
0
    def create(self):
        """
        Adds a new dataset dynamically through a POST request
        """

        # User must be authenticated so we should have a user object in
        # c.account, if not abort with error message
        if not c.account:
            abort(status_code=400, detail='user not authenticated')

        # Check if the params are there ('metadata', 'csv_file')
        if len(request.params) != 2:
            abort(status_code=400, detail='incorrect number of params')

        metadata = request.params['metadata'] \
            if 'metadata' in request.params \
            else abort(status_code=400, detail='metadata is missing')

        csv_file = request.params['csv_file'] \
            if 'csv_file' in request.params \
            else abort(status_code=400, detail='csv_file is missing')

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(metadata))
        except:
            abort(status_code=400, detail='JSON model could not be parsed')
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            abort(status_code=400, detail='Model is not well formed')
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = True  # Default value
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account, url=csv_file)

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == csv_file:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))
Example #5
0
    def load_with_model_and_csv(self, metadata, csv_file, private):
        """
        Load a dataset using a metadata model file and a csv file
        """

        if metadata is None:
            response.status = 400
            return to_jsonp({'errors': 'metadata is missing'})

        if csv_file is None:
            response.status = 400
            return to_jsonp({'errors': 'csv_file is missing'})

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(metadata))
        except:
            response.status = 400
            return to_jsonp({'errors': 'JSON model could not be parsed'})
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            response.status = 400
            return to_jsonp({'errors': 'Model is not well formed'})
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = private
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account,
                        url=csv_file)

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == csv_file:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))
Example #6
0
    def load_with_model_and_csv(self, metadata, csv_file, private):
        """
        Load a dataset using a metadata model file and a csv file
        """

        if metadata is None:
            response.status = 400
            return to_jsonp({'errors': 'metadata is missing'})

        if csv_file is None:
            response.status = 400
            return to_jsonp({'errors': 'csv_file is missing'})

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(metadata))
        except:
            response.status = 400
            return to_jsonp({'errors': 'JSON model could not be parsed'})
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            response.status = 400
            return to_jsonp({'errors': 'Model is not well formed'})
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = private
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account, url=csv_file)

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == csv_file:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))
Example #7
0
    def create(self):
        """
        Adds a new dataset dynamically through a POST request
        """

        # User must be authenticated so we should have a user object in
        # c.account, if not abort with error message
        if not c.account:
            abort(status_code=400, detail='user not authenticated')


        # Parse the loading api parameters to get them into the right format
        parser = LoadingAPIParamParser(request.params)
        params, errors = parser.parse()

        if errors:
            response.status = 400
            return to_jsonp({'errors': errors})

        if params['metadata'] is None:
            response.status = 400
            return to_jsonp({'errors': 'metadata is missing'})

        if params['csv_file'] is None:
            response.status = 400
            return to_jsonp({'errors': 'csv_file is missing'})

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(params['metadata']))
        except:
            response.status = 400
            return to_jsonp({'errors': 'JSON model could not be parsed'})
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            response.status = 400
            return to_jsonp({'errors': 'Model is not well formed'})
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = params['private']
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset, creator=c.account,
                        url=params['csv_file'])

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == params['csv_file']:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))
Example #8
0
    def create(self):
        """
        Adds a new dataset dynamically through a POST request
        """

        # User must be authenticated so we should have a user object in
        # c.account, if not abort with error message
        if not c.account:
            abort(status_code=400, detail='user not authenticated')

        # Parse the loading api parameters to get them into the right format
        parser = LoadingAPIParamParser(request.params)
        params, errors = parser.parse()

        if errors:
            response.status = 400
            return to_jsonp({'errors': errors})

        if params['metadata'] is None:
            response.status = 400
            return to_jsonp({'errors': 'metadata is missing'})

        if params['csv_file'] is None:
            response.status = 400
            return to_jsonp({'errors': 'csv_file is missing'})

        # We proceed with the dataset
        try:
            model = json.load(urllib2.urlopen(params['metadata']))
        except:
            response.status = 400
            return to_jsonp({'errors': 'JSON model could not be parsed'})
        try:
            log.info("Validating model")
            model = validate_model(model)
        except Invalid as i:
            log.error("Errors occured during model validation:")
            for field, error in i.asdict().items():
                log.error("%s: %s", field, error)
            response.status = 400
            return to_jsonp({'errors': 'Model is not well formed'})
        dataset = Dataset.by_name(model['dataset']['name'])
        if dataset is None:
            dataset = Dataset(model)
            require.dataset.create()
            dataset.managers.append(c.account)
            dataset.private = params['private']
            db.session.add(dataset)
        else:
            require.dataset.update(dataset)

        log.info("Dataset: %s", dataset.name)
        source = Source(dataset=dataset,
                        creator=c.account,
                        url=params['csv_file'])

        log.info(source)
        for source_ in dataset.sources:
            if source_.url == params['csv_file']:
                source = source_
                break
        db.session.add(source)
        db.session.commit()

        # Send loading of source into celery queue
        load_source.delay(source.id)
        return to_jsonp(dataset_apply_links(dataset.as_dict()))