def validate_all(taskname='validate_all', username=None): """Import all currently available configurations. This method is a spike to see whether the import of water balance configurations actually works. As such, it is clearly a work in progress: - there are no unit tests; - it only supports water balance configurations; - dbf files are extracted to a hard-coded directory; - dbf files are not removed after the import; - zip files are not removed after the import; - there is no error handling. """ logger = logging.getLogger(__name__) handler = get_handler(taskname=taskname, username=username) logger.addHandler(handler) retriever = create_configurations_retriever() for configuration in retriever.retrieve_configurations(): zip_file = ZipFile(configuration.zip_file_path) zip_file.extract('aanafvoer_waterbalans.dbf', '/tmp') zip_file.extract('grondwatergebieden.dbf', '/tmp') zip_file.extract('pumpingstations.dbf', '/tmp') dbfimporter = DBFImporter() dbfimporter.logger = logger dbfimporter.fews_meta_info = configuration.meta_info dbfimporter.areas_filepath = '/tmp/aanafvoer_waterbalans.dbf' dbfimporter.buckets_filepath = '/tmp/grondwatergebieden.dbf' dbfimporter.structures_filepath = '/tmp/pumpingstations.dbf' dbfimporter.import_dbf() logger.removeHandler(handler)
def validate_wbconfigurations(taskname="", username=None, levelno=20, data_set=None, configtype=None): """ Import wb areaconfigurations from dbf using validation configurations. """ logger = logging.getLogger(taskname) logger.info("Start validation of wbconfigurations for '%s'." % data_set) remove_rejected_configurations(data_set, configtype, logger) v_configs = ConfigurationToValidate.objects.filter( data_set__name__iexact=data_set, config_type=configtype, action=ConfigurationToValidate.VALIDATE) v_configs = v_configs.exclude(file_path=None) validated = 0 failed = 0 for v_config in v_configs: dbfimporter = DBFImporter(logger) dbfimporter.fews_meta_info = v_config.fews_meta_info dbfimporter.areas_filepath = v_config.area_dbf dbfimporter.buckets_filepath = v_config.grondwatergebieden_dbf dbfimporter.structures_filepath = v_config.pumpingstations_dbf logger.debug( "Start validation of 'aanafvoergebied' ident '%s'." % v_config.area.ident) status = dbfimporter.import_areaconfigurations('AreaConfiguration', v_config) if isinstance(status, tuple) and status[0]: logger.debug("Start validation of 'grondwatergebieden'.") status = dbfimporter.import_buckets('Bucket', v_config) if isinstance(status, tuple) and status[0]: logger.debug("Start validation of 'kunstwerken'.") status = dbfimporter.import_structures('Structure', v_config) if isinstance(status, tuple) and status[0]: ConfigurationToValidate.objects.get(id=v_config.id).delete() validated = validated + 1 logger.debug("Validated with SUCCESS.") else: v_config.action = ConfigurationToValidate.KEEP if isinstance(status, tuple) and len(status) > 1: v_config.action_log = status[1][:256] else: v_config.action_log = "Error ...." v_config.save() failed = failed + 1 logger.debug("Validated with ERRORS.") logger.info("Succeed=%s, Failed=%s." % (validated, failed)) logger.info("End validation.")
def import_dbf(fews_meta_info=None, areas_filepath=None, buckets_filepath=None, structures_filepath=None, taskname="", username=None, levelno=20): """Import a waterbalance configuration from dbf. This function is provided for convenience only. It allows us to test the waterbalance configuration import without the need of a ConfigurationToValidate. """ handler = get_handler(taskname=taskname, username=username) logger = logging.getLogger(taskname) logger.addHandler(handler) logger.setLevel(int(levelno)) dbfimporter = DBFImporter() dbfimporter.fews_meta_info = fews_meta_info dbfimporter.areas_filepath = areas_filepath dbfimporter.buckets_filepath = buckets_filepath dbfimporter.structures_filepath = structures_filepath # Enable lizard_history logging by starting a fake request try: user = User.objects.get(username=username) except (User.DoesNotExist, User.MultipleObjectsReturned): user = None utils.start_fake_request(user=user) try: dbfimporter.import_dbf() finally: # End the fake request, so that lizard_history will log the changes utils.end_fake_request() logger.removeHandler(handler) return "<<import dbf>>"