Esempio n. 1
0
    def resolve(self, deferred):
        """
        Takes a deferred importer object which sets the appropriate
        parser and delegates to it's resolve method.
        """

        logging.info('Executing BaseImporter.resolve')

        # get the parser class key
        klass = deferred['class']

        if not klass:
            raise Exception('Deferred object missing class key: %s' % klass)

        # get the parser class and resolve the deferred object
        self.parser = get_parser(klass).resolve(deferred)
        # return self for fluency's sake
        return self
Esempio n. 2
0
    def _get_parser_from_post(self):
        """Extract required metadata for a dataset from request.POST."""

        logging.info('Executing BaseImporter._get_parser_from_post')

        VALS_DELIMITER = settings.OPENBUDGETS_IMPORT_INTRA_FIELD_DELIMITER

        container_object_dict = {}
        parser_key = self.post_data.get('type', '')
        attributes = self.post_data
        del attributes['type']

        logging.info('BaseImporter._get_parser_from_post :: parser_key:' + parser_key)

        # get the appropriate parser class
        parser = get_parser(parser_key)

        logging.info('BaseImporter._get_parser_from_post :: parser.container_model:')
        logging.info(parser.container_model())

        logging.info('BaseImporter._get_parser_from_post :: attributes.items:')
        logging.info(attributes.items())

        for k, v in attributes.items():

            logging.info('BaseImporter._get_parser_from_post :: attributes key-value pair:')
            logging.info(k, v)

            try:
                getattr(parser.container_model(), k)
            except AttributeError as e:
                raise e

            # if the value is delimited, it is an m2m related field
            if VALS_DELIMITER in v:
                v = tuple(v.split(VALS_DELIMITER))

            container_object_dict[k] = v

            # return instantiated parser
            return parser(container_object_dict)

        else:
            raise Exception('No attributes given in meta data.')
Esempio n. 3
0
def save_import(deferred, email):

    from openbudgets.apps.transport.incoming.importers.tablibimporter import \
        TablibImporter

    importer = TablibImporter()
    saved = importer.resolve(deferred).save()

    sender = settings.EMAIL_HOST_USER
    recipient = email

    container = deferred['container']
    name = container.get('name', '')
    if not name:
        name = get_parser(deferred['class']).container_model._meta.verbose_name + ' of '
        entity = container.get('entity', None)
        period = container.get('period_start')
        if entity:
            try:
                entity_name = getattr(entity, 'name')
            except AttributeError:
                entity_name = entity.get('name')
            name += entity_name
        else:
            name += 'unknown'
        name += ' for ' + period

    if saved:
        subject = _('[OPEN BUDGETS]: Data import success')
        message = _('The data import succeeded for ' + name)

    else:
        subject = _('[OPEN BUDGETS]: Data import failure')
        message = _('The data import failed for ' + name)

    return send_mail(subject, message, sender, [recipient], fail_silently=True)
Esempio n. 4
0
    def _get_parser_from_filename(self):
        """Extract required metadata for a dataset from the filename.

        This is used for non-interactive importing of datasets from files.

        FILENAME FORMAT:

        TYPE;CONTAINER_OBJECT_KWARGS.EXTENSION

        The first positional argument is always the type (Django model name),
        and the type must be supported by a parser.

        A keyword argument-like pattern is used for the attributes of the
        container object.

        Arguments are separated by:

        settings.OPENBUDGETS_IMPORT_FIELD_DELIMITER

        defaults to ','.

        Multiple values for keys are separated by:

        settings.OPENBUDGETS_IMPORT_INTRA_FIELD_DELIMITER

        defaults to '|'.

        EXAMPLES
        --------

        TEMPLATE:
        template,name=israel-municipality,divisions=4|5|6,period_start=2001-01-10.csv
        Each row in the file is a node in the template.

        SHEET:
        sheet,entity=slug,period_start=2001-01-01,period_end=2001-12-31.csv
        Each row in the file is an item in the budget.

        """

        logging.info('Executing BaseImporter._get_parser_from_filename')

        ARGS_DELIMITER = settings.OPENBUDGETS_IMPORT_FIELD_DELIMITER
        VALS_DELIMITER = settings.OPENBUDGETS_IMPORT_INTRA_FIELD_DELIMITER

        # an empty dict to populate with data for our container object
        container_object_dict = {}

        # an empty dict to work with while creating the final container dict
        unprocessed_arguments_dict = {}

        # get our data string from the filename
        keys, ext = os.path.splitext(unicode(self.sourcefile))

        # first, split the parser key from the container_object keys
        arguments = keys.split(ARGS_DELIMITER)

        # get the appropriate parser class
        parser_key = arguments[0]
        parser = get_parser(parser_key)

        for argument in arguments[1:]:

            argument_key, argument_value = argument.split('=')

            if VALS_DELIMITER in argument_value:
                argument_value = tuple(argument_value.split(VALS_DELIMITER))

            # the arguments we got, now as a dictionary
            unprocessed_arguments_dict[argument_key] = argument_value

        # some special conditions, depending on type
        if parser_key == 'sheet':
            unprocessed_arguments_dict['entity'] = Entity.objects.get(
                slug=unprocessed_arguments_dict['entity'])

        # remove non-valid arguments
        for k in unprocessed_arguments_dict.copy():
            if not k in parser.CONTAINER_ATTRIBUTES:
                del unprocessed_arguments_dict[k]

        # update the empty container_object_dict with valid arguments
        container_object_dict.update(unprocessed_arguments_dict)

        # return instantiated parser
        return parser(container_object_dict)