def make_layers(): layers = [] for level in (u'HIG', u'MED', u'LOW'): layer = Layer() layer.hazardlevel = HazardLevel.get(level) layer.return_period = 1 layer.hazardunit = u'm' layer.data_lastupdated_date = datetime.now() layer.metadata_lastupdated_date = datetime.now() layer.geonode_id = new_geonode_id() layer.download_url = u'http://something' layer.calculation_method_quality = 5 layer.scientific_quality = 1 layer.local = False layer.downloaded = True layers.append(layer) return layers
def harvest_layer(object, dry_run=False): title = object['title'] ''' typename = urllib.unquote( object['distribution_url'].split('/').pop()) # print title ''' hazardset_id = object['hazard_set'] if not hazardset_id: logger.info('{} - hazard_set is empty'.format(title)) return False hazard_type = object['hazard_type'] if not hazard_type: logger.warning('{} - hazard_type is empty'.format(title)) return False hazardtype = hazardtype_from_geonode(hazard_type) if hazardtype is None: logger.warning('{} - hazard_type not supported: {}' .format(title, hazard_type)) return False type_settings = settings['hazard_types'][hazardtype.mnemonic] preprocessed = 'values' in type_settings ''' csw_wkt_geometry = object['csw_wkt_geometry'] bounds = wkt_loads(csw_wkt_geometry).bounds # print ' bounds :', bounds # TODO: minimum global bbox should be defined in settings minimum_global_bounds = (-175, -45, 175, 45) from shapely.geometry import box local = not box(bounds).contains(box(minimum_global_bounds)) ''' local = 'GLOBAL' not in hazardset_id ''' local = ( bounds[0] > -175 or bounds[1] > -45 or bounds[2] < 175 or bounds[3] < 45) ''' mask = False if preprocessed is True: hazardlevel = None hazard_unit = None if object['hazard_period']: logger.info('{} - Has a return period'.format(title)) return False hazard_period = None else: hazard_period = int(object['hazard_period']) hazardlevel = None for level in (u'LOW', u'MED', u'HIG'): return_periods = type_settings['return_periods'][level] if isinstance(return_periods, list): if (hazard_period >= return_periods[0] and hazard_period <= return_periods[1]): hazardlevel = HazardLevel.get(level) break else: if hazard_period == return_periods: hazardlevel = HazardLevel.get(level) if ('mask_return_period' in type_settings and hazard_period == type_settings['mask_return_period']): mask = True if hazardlevel is None and not mask: logger.info('{} - No corresponding hazard_level'.format(title)) return False hazard_unit = object['hazard_unit'] if hazard_unit == '': logger.info('{} - hazard_unit is empty'.format(title)) return False if object['srid'] != 'EPSG:4326': logger.info('{} - srid is different from "EPSG:4326"' .format(title)) return False data_update_date = object['data_update_date'] if not data_update_date: logger.warning('{} - data_update_date is empty'.format(title)) # TODO: Restore bypassed constraint to get Volcanic data # return False data_update_date = datetime.now() metadata_update_date = object['metadata_update_date'] if not metadata_update_date: logger.warning('{} - metadata_update_date is empty'.format(title)) # return False metadata_update_date = datetime.now() calculation_method_quality = object['calculation_method_quality'] if not calculation_method_quality: logger.warning('{} - calculation_method_quality is empty' .format(title)) return False calculation_method_quality = int(float(calculation_method_quality)) scientific_quality = object['scientific_quality'] if not scientific_quality: logger.warning('{} - scientific_quality is empty'.format(title)) return False scientific_quality = int(float(scientific_quality)) download_url = object['download_url'] if not download_url: logger.warning('{} - download_url is empty'.format(title)) return False hazardset = DBSession.query(HazardSet).get(hazardset_id) if hazardset is None: logger.info('{} - Create new hazardset {}' .format(title, hazardset_id)) hazardset = HazardSet() hazardset.id = hazardset_id hazardset.hazardtype = hazardtype hazardset.data_lastupdated_date = data_update_date hazardset.metadata_lastupdated_date = metadata_update_date DBSession.add(hazardset) else: # print ' Hazardset {} found'.format(hazardset_id) pass layer = DBSession.query(Layer) \ .filter(Layer.hazardset_id == hazardset_id) if hazardlevel is not None: layer = layer.filter(Layer.hazardlevel_id == hazardlevel.id) if mask: layer = layer.filter(Layer.mask.is_(True)) layer = layer.first() if layer is None: layer = Layer() logger.info('{} - Create new Layer {}'.format(title, title)) DBSession.add(layer) else: if object['id'] == layer.geonode_id: # TODO: metadata change return False if hazard_period > layer.return_period: logger.info('{} - Use preferred return period {}' .format(title, layer.return_period)) return False logger.info('{} - Replace layer for level {}' .format(title, hazardlevel.mnemonic)) layer.hazardset = hazardset layer.hazardlevel = hazardlevel layer.mask = mask layer.return_period = hazard_period layer.hazardunit = hazard_unit layer.data_lastupdated_date = data_update_date layer.metadata_lastupdated_date = metadata_update_date layer.geonode_id = object['id'] layer.download_url = download_url # TODO: retrieve quality attributes layer.calculation_method_quality = calculation_method_quality layer.scientific_quality = scientific_quality layer.local = local DBSession.flush() return True