def _decorated(*args, **kwargs): # Set up logging handler = get_handler( username=kwargs.get('username', None), taskname=kwargs.get('taskname', None)) logger = logging.getLogger(kwargs.get('taskname', __name__)) logger.addHandler(handler) logger.setLevel(kwargs.get('loglevel') or 20) try: with transaction.commit_on_success(): result = the_func(*args, **kwargs) if result is None: # you can provide your own result result = 'ok' except: logger.error('Exception') for exception_line in traceback.format_exc().split('\n'): logger.error(exception_line) result = 'failure' # Remove logging handler. Note that the handler is not removed if the_func crashes. if handler.task_execution: handler.task_execution.result = result handler.task_execution.dt_finish = datetime.datetime.now() handler.task_execution.save() logger.removeHandler(handler) return result
def sync_layers_with_wmsserver( synctask=None, all=False, username=None, taskname=None, loglevel=20): # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task if all: tasks = SyncTask.objects.all() logger.info('All sync tasks') else: tasks = [SyncTask.objects.get(synctask), ] logger.info('Task: %s' % synctask) for task in tasks: try: perform_sync_task(task=task) except Exception as e: logger.error('Something went wrong performing task %s' % task) print e continue # Remove logging handler logger.removeHandler(handler) return 'OK'
def validate_all(taskname='validate_all', username=None): """Import all currently available configurations. This method is a spike to see whether the import of water balance configurations actually works. As such, it is clearly a work in progress: - there are no unit tests; - it only supports water balance configurations; - dbf files are extracted to a hard-coded directory; - dbf files are not removed after the import; - zip files are not removed after the import; - there is no error handling. """ logger = logging.getLogger(__name__) handler = get_handler(taskname=taskname, username=username) logger.addHandler(handler) retriever = create_configurations_retriever() for configuration in retriever.retrieve_configurations(): zip_file = ZipFile(configuration.zip_file_path) zip_file.extract('aanafvoer_waterbalans.dbf', '/tmp') zip_file.extract('grondwatergebieden.dbf', '/tmp') zip_file.extract('pumpingstations.dbf', '/tmp') dbfimporter = DBFImporter() dbfimporter.logger = logger dbfimporter.fews_meta_info = configuration.meta_info dbfimporter.areas_filepath = '/tmp/aanafvoer_waterbalans.dbf' dbfimporter.buckets_filepath = '/tmp/grondwatergebieden.dbf' dbfimporter.structures_filepath = '/tmp/pumpingstations.dbf' dbfimporter.import_dbf() logger.removeHandler(handler)
def test_task(username=None, db_name=None, taskname=None): """ Test task """ handler = get_handler(username=username, taskname=taskname) logger = logging.getLogger(__name__) logger.addHandler(handler) logger.setLevel(20) logger.info('I did my job') return 'OK'
def workspace_update_trackrecords(username=None, taskname=None, loglevel=20): """ Create or replace trackrecordslayers in correct workspace """ # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task layerworkspace = LayerWorkspace.objects.get(slug='p_map') layerworkspace.layers.clear() LayerWorkspaceItem.objects.create( layer_workspace=layerworkspace, layer=Layer.objects.get(slug='p-totaal-in-bodem'), ) LayerWorkspaceItem.objects.create( layer_workspace=layerworkspace, layer=Layer.objects.get(slug='witte-waas-gebieden'), ) logger.info('Replaced P-layer') layerworkspace = LayerWorkspace.objects.get(slug='po4_map') layerworkspace.layers.clear() LayerWorkspaceItem.objects.create( layer_workspace=layerworkspace, layer=Layer.objects.get(slug='po4-in-bodemvocht'), ) LayerWorkspaceItem.objects.create( layer_workspace=layerworkspace, layer=Layer.objects.get(slug='witte-waas-gebieden'), ) logger.info('Replaced PO4-layer') layerworkspace = LayerWorkspace.objects.get(slug='aqmad_map') layerworkspace.layers.clear() LayerWorkspaceItem.objects.create( layer_workspace=layerworkspace, layer=Layer.objects.get(slug='aqmad-water-ptot'), ) LayerWorkspaceItem.objects.create( layer_workspace=layerworkspace, layer=Layer.objects.get(slug='witte-waas-gebieden'), ) logger.info('Replaced adqmad PO4-layer') # Remove logging handler logger.removeHandler(handler) return 'OK'
def sync_fewsnorm(username=None, db_name=None, taskname=None): """ The fewsnorm sync function Option data_set_name """ #logger = sync_fewsnorm.get_logger() handler = get_handler(username=username, taskname=taskname) logger = logging.getLogger(FEWSNORM_LOG_NAME) logger.addHandler(handler) logger.setLevel(20) sources = get_sources(db_name) if not sources: logger.info("No databases selected. Check your database " "settings and db_name (if provided) .") for source in sources: logger.info('Updating %s...' % source) logger.debug( 'Updating ParameterCache for fewsnorm %s...', source.name) parameters = source.sync_parameter_cache() logger.debug( 'Updating ModuleCache for fewsnorm %s...', source.name) modules = source.sync_module_cache() logger.debug( 'Updating TimeStepCache for fewsnorm %s...', source.name) time_steps = source.sync_time_step_cache() logger.debug( 'Updating GeoLocationCache for fewsnorm %s...', source.name) locations = source.sync_location_cache() # from lizard_fewsnorm.models import GeoLocationCache # locations = dict([(l.ident, l) for l in GeoLocationCache.objects.filter(fews_norm_source=source)]) logger.debug( 'Updating QualifierSetCache for fewsnorm %s...', source.name) qualifier_sets = source.sync_qualifier_set_cache() logger.debug( 'Updating TimeSeriesCache for fewsnorm %s...', source.name) source.sync_time_series_cache( locations, parameters, modules, time_steps, qualifier_sets) logger.removeHandler(handler) return 'OK'
def rebuild_jdbc_cache_task(username=None, db_name=None, taskname=None, *args, **options): """ Rebuild filter cache for fewsjdbc. Options can be timeout deep """ handler = get_handler(username=username, taskname=taskname) logger = logging.getLogger('rebuild_jdbc_cache') logger.addHandler(handler) logger.setLevel(20) rebuild_jdbc_cache(logger, *args, **options)
def workspace_update_minimap(username=None, taskname=None, loglevel=20): """ Add an area layer with a special style. """ # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task MINIMAP_LAYER_SLUG_KRW = 'minimap-krw' MINIMAP_LAYERWORKSPACE_SLUG_KRW = 'minimap-krw' MINIMAP_LAYER_SLUG_AREA = 'minimap-area' MINIMAP_LAYERWORKSPACE_SLUG_AREA = 'minimap-area' # For krw minimap _create_single_layer_workspace( layerworkspace_template_slug='watersysteemkaart', layerworkspace_slug=MINIMAP_LAYERWORKSPACE_SLUG_KRW, layerworkspace_name='MiniMap KRW', layer_template_slug='krw_waterlichaam', layer_style='vss_red_on_gray_line,vss_red_on_gray', layer_slug=MINIMAP_LAYER_SLUG_KRW, layer_name='MiniMap KRW', defaults={ 'use_location_filter': True, 'location_filter': {'key': 'env', 'tpl': 'ident:{id}'}, 'is_local_server': True, 'source_ident': 'workspace-update-command', } ) # For area minimap _create_single_layer_workspace( layerworkspace_template_slug='watersysteemkaart', layerworkspace_slug=MINIMAP_LAYERWORKSPACE_SLUG_AREA, layerworkspace_name='MiniMap gebieden', layer_template_slug='witte-waas-gebieden', layer_style='vss_red_on_gray', layer_slug=MINIMAP_LAYER_SLUG_AREA, layer_name='MiniMap gebieden', ) # Remove logging handler logger.removeHandler(handler) return 'OK'
def prepare_configurations_as_task(taskname, levelno=20, username=None): """Prepare the configurations for validation. This task has the same functionality as function ``prepare_configurations`` """ logger = logging.getLogger(taskname) handler = get_handler(taskname=taskname, username=None) logger.addHandler(handler) logger.setLevel(levelno) logger.info('Start the preparations of configurations') config_store = ConfigurationStore() config_store.logger = logger config_store.supply() logger.info('END PREPARATION') logger.removeHandler(handler)
def rebuild_unblobbed_cache(username=None, db_name=None, taskname=None): """ Populate fews unblobbed cache for better user experience """ handler = get_handler(username=username, taskname=taskname) logger = logging.getLogger(__name__) logger.addHandler(handler) logger.setLevel(20) logger.info('Processing filter tree...') fews_filters(ignore_cache=True) logger.info('Processing Timeserie.has_data_dict...') Timeserie.has_data_dict(ignore_cache=True) logger.info('Processing filters...') for f in Filter.objects.all(): f.parameters() logger.info('Finished successfully.') return 'OK'
def import_dbf(fews_meta_info=None, areas_filepath=None, buckets_filepath=None, structures_filepath=None, taskname="", username=None, levelno=20): """Import a waterbalance configuration from dbf. This function is provided for convenience only. It allows us to test the waterbalance configuration import without the need of a ConfigurationToValidate. """ handler = get_handler(taskname=taskname, username=username) logger = logging.getLogger(taskname) logger.addHandler(handler) logger.setLevel(int(levelno)) dbfimporter = DBFImporter() dbfimporter.fews_meta_info = fews_meta_info dbfimporter.areas_filepath = areas_filepath dbfimporter.buckets_filepath = buckets_filepath dbfimporter.structures_filepath = structures_filepath # Enable lizard_history logging by starting a fake request try: user = User.objects.get(username=username) except (User.DoesNotExist, User.MultipleObjectsReturned): user = None utils.start_fake_request(user=user) try: dbfimporter.import_dbf() finally: # End the fake request, so that lizard_history will log the changes utils.end_fake_request() logger.removeHandler(handler) return "<<import dbf>>"
def sync_layers_ekr( slug='vss_area_value', username=None, taskname=None, loglevel=20): """ Actually: sync_layers for ekr, esf and measures. The task adds Layer objects with correct cql filters. The cql filternames are predefined as ValueType elsewhere, such as in vss.tasks. """ # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task source_ident = 'lizard-layers::%s' % slug layer = Layer.objects.get(slug=slug) original_tags = layer.tags.all() logger.info('template: %s' % layer) tag, _ = Tag.objects.get_or_create(slug=slug) logger.info('tag: %s' % tag) logger.debug('Invalidating existing layers...') existing_layers = dict( (l.slug, l) for l in Layer.objects.filter(source_ident=source_ident)) for existing_layer in existing_layers.values(): # Invalidate first and remove tags existing_layer.valid = False existing_layer.tags.clear() existing_layer.save() count_update, count_new = 0, 0 group_tag_ekr = 'ekr-layers' group_tag_esf = 'esf-layers' group_tag_measure_status = 'measure-status-layers' name_cql_style = ( ('EKR VIS', "name = 'EKR-VIS'", 'vss_ekr_value', group_tag_ekr), ('EKR FYTOPL', "name = 'EKR-FYTOPL'", 'vss_ekr_value', group_tag_ekr), ('EKR MAFAUNA', "name = 'EKR-MAFAUNA'", 'vss_ekr_value', group_tag_ekr), ('EKR OVWFLORA', "name = 'EKR-OVWFLORA'", 'vss_ekr_value', group_tag_ekr), ('EKR Minst gunstig', "name = 'EKR-ONGUNSTIG'", 'vss_ekr_value', group_tag_ekr), ('EKR Doelstelling', "name = 'EKR-DOELSCORE'", 'vss_ekr_score', group_tag_ekr), ('ESF 1', "name = 'ESF-1'", 'vss_esf', group_tag_esf), ('ESF 2', "name = 'ESF-2'", 'vss_esf', group_tag_esf), ('ESF 3', "name = 'ESF-3'", 'vss_esf', group_tag_esf), ('ESF 4', "name = 'ESF-4'", 'vss_esf', group_tag_esf), ('ESF 5', "name = 'ESF-5'", 'vss_esf', group_tag_esf), ('ESF 6', "name = 'ESF-6'", 'vss_esf', group_tag_esf), ('ESF 7', "name = 'ESF-7'", 'vss_esf', group_tag_esf), ('ESF 8', "name = 'ESF-8'", 'vss_esf', group_tag_esf), ('ESF 9', "name = 'ESF-9'", 'vss_esf', group_tag_esf), ('ESF STATUS', "name = 'ESF-STATUS'", 'vss_esf', group_tag_esf), ('Maatregel status', "name = 'MEASURE-STATUS'", 'vss_measure_status', group_tag_measure_status), ('Maatregel status planning', "name = 'MEASURE-STATUS-PLANNING'", 'vss_measure_status', group_tag_measure_status), ('Aantal maatregelen aan- afvoergebied', "name = 'MEASURE-COUNT'", 'vss_measure_count', group_tag_measure_status), ('Aantal krw maatregelen', "name = 'MEASURE-COUNT-KRW'", 'vss_measure_count', group_tag_measure_status), ('Aantal uitgevoerde maatregelen aan- afvoergebied', "name = 'MEASURE-COUNT-FINISHED'", 'vss_measure_count', group_tag_measure_status), ('Aantal uitgevoerde krw maatregelen', "name = 'MEASURE-COUNT-FINISHED-KRW'", 'vss_measure_count', group_tag_measure_status), ) for name, cql, style, group_tag in name_cql_style: instance_slug = slugify(name) if instance_slug in existing_layers: # Update existing, the old existing tags have been # removed already. new_layer = existing_layers[instance_slug] logger.debug('Update: %s' % instance_slug) new_layer.data_set = layer.data_set count_update += 1 else: # New logger.debug('New: %s' % instance_slug) new_layer = deepcopy(layer) new_layer.slug = instance_slug new_layer.id = None count_new += 1 new_layer.filter = cql # Note that the same name can occur multiple times, but # with different mod, qua and/or stp. new_layer.name = name new_layer.source_ident = source_ident new_layer.valid = True new_layer.is_local_server = True new_layer.is_clickable = layer.is_local_server new_layer.js_popup_class = layer.js_popup_class new_layer.request_params = simplejson.dumps(dict(styles=style)) new_layer.save() new_layer.tags.add(tag) for original_tag in original_tags: new_layer.tags.add(original_tag) group_tag, _ = Tag.objects.get_or_create( slug=group_tag, ) new_layer.tags.add(group_tag) logger.info('new %d items' % count_new) logger.info('updated %d items' % count_update) # Remove logging handler logger.removeHandler(handler) return 'OK'
def sync_ekr(username=None, taskname=None, dataset=None, loglevel=20): # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) logger.info('sync_ekr') # Actual code to do the task value_type_worst = ValueType.objects.get(name='EKR-ONGUNSTIG') value_type_score = ValueType.objects.get(name='EKR-DOELSCORE') parameter_types = list(ParameterType.objects.all()) areas = Area.objects.filter( area_class=Area.AREA_CLASS_KRW_WATERLICHAAM, ) if dataset: areas = areas.filter(data_set__name=dataset) logger.info('Data set: %s' % dataset) logger.info('Updating %d areas...' % areas.count()) for area in areas: logger.debug('Updating area %s...' % area) values = [] for parameter_type in parameter_types: timeseries = TimeSeriesCache.objects.filter( parametercache=parameter_type.parameter, geolocationcache__ident=area.ident, ) value = None flag = None comment = None timestamp = None if timeseries: timeserie = timeseries[0] try: event = timeserie.get_latest_event() value = event.value flag = event.flag comment = event.comment timestamp = event.timestamp except IndexError: # No events at all pass update_area_value( area=area, value=value, flag=flag, comment=comment, timestamp=timestamp, value_type=parameter_type.value_type, ) values.append({ 'parameter_type': parameter_type, 'value': value, 'flag': flag, 'comment': comment, 'timestamp': timestamp, }) try: # value_worst = min([v['value'] for v in values # if v['value'] is not None]) value_worst = min(values, key=lambda v: v['value']) except ValueError: # All ekrs are None value_worst = None if value_worst is not None: update_area_value( area=area, value=value_worst['value'], comment=value_worst['comment'], value_type=value_type_worst, ) else: update_area_value( area=area, value=None, comment=None, value_type=value_type_worst, ) judgements = _judgements(values, area) overall_judgement = _overall_judgement(judgements) update_area_value( area=area, value=overall_judgement, comment=None, value_type=value_type_score, ) logger.debug('worst: %s, overall: %s' % ( str(value_worst), overall_judgement)) logger.info('Finished') # Remove logging handler logger.removeHandler(handler) return 'OK'
def workspace_update_watersystem(username=None, taskname=None, loglevel=20): """ Reconfigure layers for the watersystem map. """ def _create_or_replace_merged_layer(name, slug, tag, layers): """ Return created layer object. Laye is created with name and slug, and layers assembled from layers field of contributing layers. Contributing layers are looked up by the same layers field. """ Layer.objects.filter(slug=slug).delete() contributing_layers = Layer.objects.filter( layers__in=layers, ) layer = Layer.objects.create( slug=slug, name=name, source_ident='workspace-update-command', server=contributing_layers[0].server, layers=','.join(l.layers for l in contributing_layers), ) layer.tags.add(tag) return layer # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task WORKSPACE_SLUG = 'watersysteemkaart' TAG_SLUG = 'basis' # Clear the layer workspace, get the tag layer_workspace = LayerWorkspace.objects.get( slug=WORKSPACE_SLUG, ) LayerWorkspaceItem.objects.filter( layer_workspace=layer_workspace, ).delete() tag = Tag.objects.get_or_create(slug=TAG_SLUG)[0] # Create layers and add to workspace # Kunstwerken basis layer = _create_or_replace_merged_layer( slug='kunstwerken-basis', name='Kunstwerken Basis', tag=tag, layers=['wsh:gemaal', 'wsh:stuw', 'wsh:sluis'], ) LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=True, index=10, ) # Kunstwerken extra layer = _create_or_replace_merged_layer( slug='kunstwerken-extra', name='Kunstwerken Extra', tag=tag, layers=[ 'wsh:vispassage', 'wsh:vaste_dam', 'wsh:sifon', 'wsh:duiker', 'wsh:coupure', 'wsh:brug', 'wsh:aquaduct', ], ) LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=False, index=20, ) # Peilgebied layer = _create_or_replace_merged_layer( slug='peilgebied-basis', name='Peilgebied', tag=tag, layers=['wsh:peilgebied'], ) LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=False, index=30, ) # Waterloop layer = _create_or_replace_merged_layer( slug='waterloop-basis', name='Waterloop', tag=tag, layers=['wsh:waterloop'], ) LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=False, index=40, ) # Oppervlake waterdeel layer = _create_or_replace_merged_layer( slug='oppervlakte-waterdeel-basis', name='Oppervlakte waterdeel', tag=tag, layers=['wsh:oppervlakte_waterdeel'], ) LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=False, index=50, ) # Waterlichaam layer = _create_or_replace_merged_layer( slug='krw_waterlichaam', name='KRW-waterlichaam', tag=tag, layers=[ 'vss:vss_krw_waterbody_polygon', 'vss:vss_krw_waterbody_linestring', ], ) LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=False, index=60, ) layer = Layer.objects.get(slug='vss_measure') layer.name = 'Maatregelen' layer.js_popup_class = 'MeasurePopup' layer.save() LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=False, index=70, ) layer = Layer.objects.get(slug='vss_annotation') layer.name = 'Analyse interpretaties' layer.js_popup_class = 'AnnotationPopup' layer.save() LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=False, index=80, ) layer = Layer.objects.get(slug='witte-waas-gebieden') layer.name = 'Masker' layer.save() LayerWorkspaceItem.objects.create( layer_workspace=layer_workspace, layer=layer, visible=True, index=90, ) logger.info('Reinstalled watersystem workspace.') # Remove logging handler logger.removeHandler(handler) return 'OK'
def workspace_update_baselayers(username=None, taskname=None, loglevel=20): """ Reconfigure layers that have is_base_layer=True """ # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task TOP10NL_LAYER_SLUG = 'top10nl' TOP10NL_TAG_SLUG = 'server_pdok-top10' SEMI_TRANSPARENT_SLUG_POSTFIX = '-semi-transparent' SEMI_TRANSPARENT_NAME_POSTFIX = ' (semitransparant)' # Get open streetmap, rename if necessary try: osm = Layer.objects.get(name='openstreetmap') osm.name = 'OpenStreetMap' osm.save() except: osm = Layer.objects.get(name='OpenStreetMap') # Remove anything except osm Layer.objects.filter( is_base_layer=True, ).exclude( pk=osm.pk, ).update(is_base_layer=False) # Remove old baselayer(s) for the top10nl if it exists Layer.objects.filter(slug=TOP10NL_LAYER_SLUG).delete() # Add a baselayer for the top10nl tag = Tag.objects.get(slug=TOP10NL_TAG_SLUG) top10_layers = tag.layer_set.all() new_layer = top10_layers[0] new_layer.name = 'Top10NL' new_layer.slug = TOP10NL_LAYER_SLUG new_layer.is_base_layer = True new_layer.source_ident = 'workspace-update-command', new_layer.layers = ','.join(l.layers for l in top10_layers) new_layer.source_ident = None new_layer.pk = None # We want a new layer. new_layer.save() logger.info('Created default baselayers.') # Add or replace baselayers with 50% opacity base_layers = Layer.objects.filter(is_base_layer=True) Layer.objects.filter( slug__in=[b.slug + SEMI_TRANSPARENT_SLUG_POSTFIX for b in base_layers], ).delete() for b in base_layers: options = simplejson.loads(b.options) options.update(opacity=0.5) b.pk = None # Clone the layer b.source_ident = 'workspace-update-command' b.slug += SEMI_TRANSPARENT_SLUG_POSTFIX b.name += SEMI_TRANSPARENT_NAME_POSTFIX b.options = simplejson.dumps(options) b.save() logger.info('Added transparent variants of default baselayers.') # Remove logging handler logger.removeHandler(handler) return 'OK'
def sync_ekr(username=None, taskname=None, dataset=None, loglevel=20): # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) logger.info('sync_ekr') # Actual code to do the task value_type_worst = ValueType.objects.get(name='EKR-ONGUNSTIG') value_type_score = ValueType.objects.get(name='EKR-DOELSCORE') parameter_types = list(ParameterType.objects.all()) areas = Area.objects.filter(area_class=Area.AREA_CLASS_KRW_WATERLICHAAM, ) if dataset: areas = areas.filter(data_set__name=dataset) logger.info('Data set: %s' % dataset) logger.info('Updating %d areas...' % areas.count()) for area in areas: logger.debug('Updating area %s...' % area) values = [] for parameter_type in parameter_types: timeseries = TimeSeriesCache.objects.filter( parametercache=parameter_type.parameter, geolocationcache__ident=area.ident, ) value = None flag = None comment = None timestamp = None if timeseries: timeserie = timeseries[0] try: event = timeserie.get_latest_event() value = event.value flag = event.flag comment = event.comment timestamp = event.timestamp except IndexError: # No events at all pass update_area_value( area=area, value=value, flag=flag, comment=comment, timestamp=timestamp, value_type=parameter_type.value_type, ) values.append({ 'parameter_type': parameter_type, 'value': value, 'flag': flag, 'comment': comment, 'timestamp': timestamp, }) try: # value_worst = min([v['value'] for v in values # if v['value'] is not None]) value_worst = min(values, key=lambda v: v['value']) except ValueError: # All ekrs are None value_worst = None if value_worst is not None: update_area_value( area=area, value=value_worst['value'], comment=value_worst['comment'], value_type=value_type_worst, ) else: update_area_value( area=area, value=None, comment=None, value_type=value_type_worst, ) judgements = _judgements(values, area) overall_judgement = _overall_judgement(judgements) update_area_value( area=area, value=overall_judgement, comment=None, value_type=value_type_score, ) logger.debug('worst: %s, overall: %s' % (str(value_worst), overall_judgement)) logger.info('Finished') # Remove logging handler logger.removeHandler(handler) return 'OK'
def sync_layers_track( slug='vss_track_records', username=None, taskname=None, loglevel=20): """ Sync layers for track records. """ # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task source_ident = 'lizard-layers::%s' % slug layer = Layer.objects.get(slug=slug) original_tags = layer.tags.all() logger.info('template: %s' % layer) tag, _ = Tag.objects.get_or_create(slug=slug) logger.info('tag: %s' % tag) logger.debug('Invalidating existing layers...') existing_layers = dict( (l.slug, l) for l in Layer.objects.filter(source_ident=source_ident)) for existing_layer in existing_layers.values(): # Invalidate first and remove tags existing_layer.valid = False existing_layer.tags.clear() existing_layer.save() count_update, count_new = 0, 0 group_tag = 'track_records' parameter_id_Ptot = ParameterCache.objects.get(ident='Ptot.bodem').id parameter_id_PO4 = ParameterCache.objects.get(ident='PO4.bodem').id parameter_id_aqmad_Ptot = ParameterCache.objects.get( ident='Ptot.z-score.water', ).id name_cql_style = ( ( 'PO4 in bodemvocht', "parameter_id = %s" % parameter_id_PO4, 'vss_track_record_PO4', ), ( 'P-totaal in bodem', "parameter_id = %s" % parameter_id_Ptot, 'vss_track_record_Ptot', ), ( 'AqMaD water Ptot', "parameter_id = %s" % parameter_id_aqmad_Ptot, 'vss_aqmad_Ptot', ), ) for name, cql, style in name_cql_style: instance_slug = slugify(name) if instance_slug in existing_layers: # Update existing, the old existing tags have been # removed already. new_layer = existing_layers[instance_slug] logger.debug('Update: %s' % instance_slug) new_layer.data_set = layer.data_set count_update += 1 else: # New logger.debug('New: %s' % instance_slug) new_layer = deepcopy(layer) new_layer.slug = instance_slug new_layer.id = None count_new += 1 new_layer.filter = cql # Note that the same name can occur multiple times, but # with different mod, qua and/or stp. new_layer.name = name new_layer.source_ident = source_ident new_layer.valid = True new_layer.is_local_server = True new_layer.is_clickable = layer.is_local_server new_layer.js_popup_class = layer.js_popup_class new_layer.request_params = simplejson.dumps(dict(styles=style)) new_layer.save() new_layer.tags.add(tag) for original_tag in original_tags: new_layer.tags.add(original_tag) group_tag, _ = Tag.objects.get_or_create( slug=group_tag, ) new_layer.tags.add(group_tag) logger.info('new %d items' % count_new) logger.info('updated %d items' % count_update) # Remove logging handler logger.removeHandler(handler) return 'OK'
def sync_layers_measure( slug='vss_measure', username=None, taskname=None, loglevel=20): # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task logger.info('start sync') source_ident = 'lizard-layers::%s' % slug layer = Layer.objects.get(slug=slug) original_tags = layer.tags.all() logger.info('template: %s' % layer) tag, _ = Tag.objects.get_or_create(slug=slug) logger.info('tag: %s' % tag) logger.debug('Invalidating existing layers...') existing_layers = dict( (l.slug, l) for l in Layer.objects.filter(source_ident=source_ident)) for existing_layer in existing_layers.values(): # Invalidate first and remove tags existing_layer.valid = False existing_layer.tags.clear() existing_layer.save() count_update, count_new = 0, 0 esf_name_template = "Maatregelen ESF%s" esf_cql_template = ("esf = %s AND (" "is_target_esf = TRUE OR " "positive = TRUE OR " "negative = TRUE)") datalist = [ # For measure layers based on type { 'group_tag': 'maatregel-type', 'cql_and_names': ( ("type like 'BE%'", 'Beheermaatregelen'), ("type like 'BR%'", 'Bronmaatregelen'), ("type like 'IM%'", 'Immissiemaatregelen'), ("type like 'IN%'", 'Inrichtingsmaatregelen'), ( "type LIKE 'G%' OR type LIKE 'S%' OR type LIKE 'RO%'", 'Overige maatregelen', ), ) }, # For measure layers based on related esf { 'group_tag': 'maatregel-esf', 'cql_and_names': [(esf_cql_template % e, esf_name_template % e) for e in range(1, 10)] }, ] for datadict in datalist: for cql, name in datadict['cql_and_names']: instance_slug = slugify(name) if instance_slug in existing_layers: # Update existing, the old existing tags have been # removed already. new_layer = existing_layers[instance_slug] logger.debug('Update: %s' % instance_slug) new_layer.data_set = layer.data_set count_update += 1 else: # New logger.debug('New: %s' % instance_slug) new_layer = deepcopy(layer) new_layer.slug = instance_slug new_layer.id = None count_new += 1 new_layer.filter = cql # Note that the same name can occur multiple times, but # with different mod, qua and/or stp. new_layer.name = name new_layer.source_ident = source_ident new_layer.valid = True new_layer.is_local_server = True new_layer.is_clickable = layer.is_local_server new_layer.js_popup_class = layer.js_popup_class new_layer.save() new_layer.tags.add(tag) for original_tag in original_tags: new_layer.tags.add(original_tag) group_tag, _ = Tag.objects.get_or_create( slug=datadict['group_tag'], ) new_layer.tags.add(group_tag) logger.info('new %d items' % count_new) logger.info('updated %d items' % count_update) # Remove logging handler logger.removeHandler(handler) return 'OK'
def sync_layers_fewsnorm( slug='vss_fews_locations', username=None, taskname=None, loglevel=20): # Set up logging handler = get_handler(username=username, taskname=taskname) logger.addHandler(handler) logger.setLevel(loglevel) # Actual code to do the task source_ident = 'fewsnorm::%s' % slug layer = Layer.objects.get(slug=slug) original_tags = layer.tags.all() logger.info('template: %s' % layer) tag, _ = Tag.objects.get_or_create(slug='fewsnorm_%s' % slug) logger.info('tag: %s' % tag) logger.debug('Invalidating existing layers...') existing_layers = dict( (l.slug, l) for l in Layer.objects.filter(source_ident=source_ident)) for existing_layer in existing_layers.values(): # Invalidate first. existing_layer.valid = False # Remove tags from many to many relationships, not delete # the tags themselves. for layer_tag in existing_layer.tags.all(): existing_layer.tags.remove(layer_tag) existing_layer.save() count_update, count_new = 0, 0 for par_mod_qua_stp in TimeSeriesCache.objects.filter(active=True).values( "parametercache__ident", "parametercache__name", "modulecache__ident", "qualifiersetcache__ident", "timestepcache__ident").distinct(): par = par_mod_qua_stp['parametercache__ident'] par_name = par_mod_qua_stp['parametercache__name'] mod = par_mod_qua_stp['modulecache__ident'] qua = par_mod_qua_stp['qualifiersetcache__ident'] stp = par_mod_qua_stp['timestepcache__ident'] instance_slug = '%s_%s_%s_%s_%s' % (slug, par, mod, qua, stp) if instance_slug in existing_layers: # Update existing, the old existing tags have been # removed already. new_layer = existing_layers[instance_slug] logger.debug('Update: %s' % instance_slug) new_layer.data_set = layer.data_set count_update += 1 else: # New logger.debug('New: %s' % instance_slug) new_layer = deepcopy(layer) new_layer.slug = instance_slug new_layer.id = None count_new += 1 layer_params = [] if par: layer_params.append("par_ident='%s'" % par) if mod: layer_params.append("mod_ident='%s'" % mod) if qua: layer_params.append("qua_ident='%s'" % qua) if stp: layer_params.append("stp_ident='%s'" % stp) new_layer.filter = ' and '.join(layer_params) # Note that the same name can occur multiple times, but # with different mod, qua and/or stp. if qua is None: new_layer.name = '%s (%s)' % (par_name, stp) else: new_layer.name = '%s %s (%s)' % (par_name, qua, stp) new_layer.name = new_layer.name[:80] new_layer.source_ident = source_ident new_layer.valid = True new_layer.is_local_server = layer.is_local_server new_layer.is_clickable = layer.is_local_server new_layer.js_popup_class = layer.js_popup_class new_layer.save() new_layer.tags.add(tag) for original_tag in original_tags: new_layer.tags.add(original_tag) if mod is not None: # add tag mod_tag, _ = Tag.objects.get_or_create(slug='%s_%s' % (tag, mod)) new_layer.tags.add(mod_tag) logger.info('new %d items' % count_new) logger.info('updated %d items' % count_update) # Remove logging handler logger.removeHandler(handler) return 'OK'