def validate_all(taskname='validate_all', username=None):
"""Import all currently available configurations.
This method is a spike to see whether the import of water balance
configurations actually works. As such, it is clearly a work in progress:
- there are no unit tests;
- it only supports water balance configurations;
- dbf files are extracted to a hard-coded directory;
- dbf files are not removed after the import;
- zip files are not removed after the import;
- there is no error handling.
"""
logger = logging.getLogger(__name__)
handler = get_handler(taskname=taskname, username=username)
logger.addHandler(handler)
retriever = create_configurations_retriever()
for configuration in retriever.retrieve_configurations():
zip_file = ZipFile(configuration.zip_file_path)
zip_file.extract('aanafvoer_waterbalans.dbf', '/tmp')
zip_file.extract('grondwatergebieden.dbf', '/tmp')
zip_file.extract('pumpingstations.dbf', '/tmp')
dbfimporter = DBFImporter()
dbfimporter.logger = logger
dbfimporter.fews_meta_info = configuration.meta_info
dbfimporter.areas_filepath = '/tmp/aanafvoer_waterbalans.dbf'
dbfimporter.buckets_filepath = '/tmp/grondwatergebieden.dbf'
dbfimporter.structures_filepath = '/tmp/pumpingstations.dbf'
dbfimporter.import_dbf()
logger.removeHandler(handler)
def validate_wbconfigurations(taskname="",
username=None,
levelno=20,
data_set=None,
configtype=None):
"""
Import wb areaconfigurations from dbf using
validation configurations.
"""
logger = logging.getLogger(taskname)
logger.info("Start validation of wbconfigurations for '%s'." % data_set)
remove_rejected_configurations(data_set, configtype, logger)
v_configs = ConfigurationToValidate.objects.filter(
data_set__name__iexact=data_set,
config_type=configtype,
action=ConfigurationToValidate.VALIDATE)
v_configs = v_configs.exclude(file_path=None)
validated = 0
failed = 0
for v_config in v_configs:
dbfimporter = DBFImporter(logger)
dbfimporter.fews_meta_info = v_config.fews_meta_info
dbfimporter.areas_filepath = v_config.area_dbf
dbfimporter.buckets_filepath = v_config.grondwatergebieden_dbf
dbfimporter.structures_filepath = v_config.pumpingstations_dbf
logger.debug(
"Start validation of 'aanafvoergebied' ident '%s'." % v_config.area.ident)
status = dbfimporter.import_areaconfigurations('AreaConfiguration', v_config)
if isinstance(status, tuple) and status[0]:
logger.debug("Start validation of 'grondwatergebieden'.")
status = dbfimporter.import_buckets('Bucket', v_config)
if isinstance(status, tuple) and status[0]:
logger.debug("Start validation of 'kunstwerken'.")
status = dbfimporter.import_structures('Structure', v_config)
if isinstance(status, tuple) and status[0]:
ConfigurationToValidate.objects.get(id=v_config.id).delete()
validated = validated + 1
logger.debug("Validated with SUCCESS.")
else:
v_config.action = ConfigurationToValidate.KEEP
if isinstance(status, tuple) and len(status) > 1:
v_config.action_log = status[1][:256]
else:
v_config.action_log = "Error ...."
v_config.save()
failed = failed + 1
logger.debug("Validated with ERRORS.")
logger.info("Succeed=%s, Failed=%s." % (validated, failed))
logger.info("End validation.")
def import_dbf(fews_meta_info=None,
areas_filepath=None,
buckets_filepath=None,
structures_filepath=None,
taskname="",
username=None,
levelno=20):
"""Import a waterbalance configuration from dbf.
This function is provided for convenience only. It allows us to test the
waterbalance configuration import without the need of a
ConfigurationToValidate.
"""
handler = get_handler(taskname=taskname, username=username)
logger = logging.getLogger(taskname)
logger.addHandler(handler)
logger.setLevel(int(levelno))
dbfimporter = DBFImporter()
dbfimporter.fews_meta_info = fews_meta_info
dbfimporter.areas_filepath = areas_filepath
dbfimporter.buckets_filepath = buckets_filepath
dbfimporter.structures_filepath = structures_filepath
# Enable lizard_history logging by starting a fake request
try:
user = User.objects.get(username=username)
except (User.DoesNotExist, User.MultipleObjectsReturned):
user = None
utils.start_fake_request(user=user)
try:
dbfimporter.import_dbf()
finally:
# End the fake request, so that lizard_history will log the changes
utils.end_fake_request()
logger.removeHandler(handler)
return "<<import dbf>>"