def create_outputs(hazardset, layers, readers): type_settings = settings['hazard_types'][hazardset.hazardtype.mnemonic] adminlevel_REG = AdminLevelType.get(u'REG') bbox = None for reader in readers.itervalues(): polygon = polygon_from_boundingbox(reader.bounds) if bbox is None: bbox = polygon else: bbox = bbox.intersection(polygon) admindivs = DBSession.query(AdministrativeDivision) \ .filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \ .filter(func.ST_Intersects(AdministrativeDivision.geom, func.ST_GeomFromText(bbox.wkt, 4326))) \ .order_by(AdministrativeDivision.id) # Needed for windowed querying current = 0 last_percent = 0 outputs = [] total = admindivs.count() logger.info(' Iterating over {} administrative divisions'.format(total)) # Windowed querying to limit memory usage limit = 1000 # 1000 records <=> 10 Mo admindivs = admindivs.limit(limit) for offset in xrange(0, total, limit): admindivs = admindivs.offset(offset) for admindiv in admindivs: current += 1 if admindiv.geom is None: logger.warning(' {}-{} has null geometry' .format(admindiv.code, admindiv.name)) continue shape = to_shape(admindiv.geom) # Try block to include admindiv.code in exception message try: if 'values' in type_settings.keys(): # preprocessed layer hazardlevel = preprocessed_hazardlevel( type_settings, layers[0], readers[0], shape) else: hazardlevel = notpreprocessed_hazardlevel( hazardset.hazardtype.mnemonic, type_settings, layers, readers, shape) except Exception as e: e.message = ("{}-{} raises an exception :\n{}" .format(admindiv.code, admindiv.name, e.message)) raise # Create output record if hazardlevel is not None: output = Output() output.hazardset = hazardset output.admin_id = admindiv.id output.hazardlevel = hazardlevel # TODO: calculate coverage ratio output.coverage_ratio = 100 outputs.append(output) # Remove admindiv from memory DBSession.expunge(admindiv) percent = int(100.0 * current / total) if percent % 10 == 0 and percent != last_percent: logger.info(' ... processed {}%'.format(percent)) last_percent = percent return outputs