def resolve(self, deferred):
"""
Takes a deferred importer object which sets the appropriate
parser and delegates to it's resolve method.
"""
logging.info('Executing BaseImporter.resolve')
# get the parser class key
klass = deferred['class']
if not klass:
raise Exception('Deferred object missing class key: %s' % klass)
# get the parser class and resolve the deferred object
self.parser = get_parser(klass).resolve(deferred)
# return self for fluency's sake
return self
def _get_parser_from_post(self):
"""Extract required metadata for a dataset from request.POST."""
logging.info('Executing BaseImporter._get_parser_from_post')
VALS_DELIMITER = settings.OPENBUDGETS_IMPORT_INTRA_FIELD_DELIMITER
container_object_dict = {}
parser_key = self.post_data.get('type', '')
attributes = self.post_data
del attributes['type']
logging.info('BaseImporter._get_parser_from_post :: parser_key:' + parser_key)
# get the appropriate parser class
parser = get_parser(parser_key)
logging.info('BaseImporter._get_parser_from_post :: parser.container_model:')
logging.info(parser.container_model())
logging.info('BaseImporter._get_parser_from_post :: attributes.items:')
logging.info(attributes.items())
for k, v in attributes.items():
logging.info('BaseImporter._get_parser_from_post :: attributes key-value pair:')
logging.info(k, v)
try:
getattr(parser.container_model(), k)
except AttributeError as e:
raise e
# if the value is delimited, it is an m2m related field
if VALS_DELIMITER in v:
v = tuple(v.split(VALS_DELIMITER))
container_object_dict[k] = v
# return instantiated parser
return parser(container_object_dict)
else:
raise Exception('No attributes given in meta data.')
def save_import(deferred, email):
from openbudgets.apps.transport.incoming.importers.tablibimporter import \
TablibImporter
importer = TablibImporter()
saved = importer.resolve(deferred).save()
sender = settings.EMAIL_HOST_USER
recipient = email
container = deferred['container']
name = container.get('name', '')
if not name:
name = get_parser(deferred['class']).container_model._meta.verbose_name + ' of '
entity = container.get('entity', None)
period = container.get('period_start')
if entity:
try:
entity_name = getattr(entity, 'name')
except AttributeError:
entity_name = entity.get('name')
name += entity_name
else:
name += 'unknown'
name += ' for ' + period
if saved:
subject = _('[OPEN BUDGETS]: Data import success')
message = _('The data import succeeded for ' + name)
else:
subject = _('[OPEN BUDGETS]: Data import failure')
message = _('The data import failed for ' + name)
return send_mail(subject, message, sender, [recipient], fail_silently=True)
def _get_parser_from_filename(self):
"""Extract required metadata for a dataset from the filename.
This is used for non-interactive importing of datasets from files.
FILENAME FORMAT:
TYPE;CONTAINER_OBJECT_KWARGS.EXTENSION
The first positional argument is always the type (Django model name),
and the type must be supported by a parser.
A keyword argument-like pattern is used for the attributes of the
container object.
Arguments are separated by:
settings.OPENBUDGETS_IMPORT_FIELD_DELIMITER
defaults to ','.
Multiple values for keys are separated by:
settings.OPENBUDGETS_IMPORT_INTRA_FIELD_DELIMITER
defaults to '|'.
EXAMPLES
--------
TEMPLATE:
template,name=israel-municipality,divisions=4|5|6,period_start=2001-01-10.csv
Each row in the file is a node in the template.
SHEET:
sheet,entity=slug,period_start=2001-01-01,period_end=2001-12-31.csv
Each row in the file is an item in the budget.
"""
logging.info('Executing BaseImporter._get_parser_from_filename')
ARGS_DELIMITER = settings.OPENBUDGETS_IMPORT_FIELD_DELIMITER
VALS_DELIMITER = settings.OPENBUDGETS_IMPORT_INTRA_FIELD_DELIMITER
# an empty dict to populate with data for our container object
container_object_dict = {}
# an empty dict to work with while creating the final container dict
unprocessed_arguments_dict = {}
# get our data string from the filename
keys, ext = os.path.splitext(unicode(self.sourcefile))
# first, split the parser key from the container_object keys
arguments = keys.split(ARGS_DELIMITER)
# get the appropriate parser class
parser_key = arguments[0]
parser = get_parser(parser_key)
for argument in arguments[1:]:
argument_key, argument_value = argument.split('=')
if VALS_DELIMITER in argument_value:
argument_value = tuple(argument_value.split(VALS_DELIMITER))
# the arguments we got, now as a dictionary
unprocessed_arguments_dict[argument_key] = argument_value
# some special conditions, depending on type
if parser_key == 'sheet':
unprocessed_arguments_dict['entity'] = Entity.objects.get(
slug=unprocessed_arguments_dict['entity'])
# remove non-valid arguments
for k in unprocessed_arguments_dict.copy():
if not k in parser.CONTAINER_ATTRIBUTES:
del unprocessed_arguments_dict[k]
# update the empty container_object_dict with valid arguments
container_object_dict.update(unprocessed_arguments_dict)
# return instantiated parser
return parser(container_object_dict)
